From 658f40c7d134c58816c6abf4054d4604a34addf6 Mon Sep 17 00:00:00 2001 From: ateneva Date: Fri, 26 Jun 2026 23:49:37 +0200 Subject: [PATCH 1/8] CU-86c2y3vu1-linting-should-run-opening-a-pr --- .github/workflows/linting-checks.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linting-checks.yml b/.github/workflows/linting-checks.yml index 9e06621..13ddc7a 100644 --- a/.github/workflows/linting-checks.yml +++ b/.github/workflows/linting-checks.yml @@ -1,12 +1,12 @@ -name: Post-Merge Checks +name: PR Linting Checks on: - push: - branches: - - main + pull_request: + types: [opened, synchronize, reopened] permissions: contents: read + pull-requests: write jobs: pre-commit-checks: From 80d4c57e8d58b137e63bae48d9bffea16cd02561 Mon Sep 17 00:00:00 2001 From: ateneva Date: Sat, 27 Jun 2026 00:00:11 +0200 Subject: [PATCH 2/8] CU-86c2y3vu1-upgrade-to-secure-sqlfluff-version --- .pre-commit-config.yaml | 6 +++--- requirements.txt | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5812b7c..e68320d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,12 +8,12 @@ repos: include: \.(yaml|yml)$ - repo: https://github.com/sqlfluff/sqlfluff - rev: 3.2.3 + rev: 4.2.0 hooks: - id: sqlfluff-lint - additional_dependencies: ['dbt-bigquery==1.8.1', 'sqlfluff-templater-dbt==3.2.3'] + additional_dependencies: ['dbt-bigquery==1.11.0', 'sqlfluff-templater-dbt==4.2.0'] - id: sqlfluff-fix - additional_dependencies: ['dbt-bigquery==1.8.1', 'sqlfluff-templater-dbt==3.2.3'] + additional_dependencies: ['dbt-bigquery==1.11.0', 'sqlfluff-templater-dbt==4.2.0'] - repo: https://github.com/dbt-checkpoint/dbt-checkpoint rev: v1.2.1 diff --git a/requirements.txt b/requirements.txt index 99e1ec1..628fd5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ dbt-core==1.11.2 dbt-bigquery==1.11.0 -sqlfluff==4.0.0 -sqlfluff-templater-dbt==4.0.0 +sqlfluff==4.2.0 +sqlfluff-templater-dbt==4.2.0 yamllint==1.35.1 pylint==3.3.1 pymarkdownlnt==0.9.24 From 1d22c6244eb8d8140e07019c47e2c8b0e98bccf6 Mon Sep 17 00:00:00 2001 From: ateneva Date: Sat, 27 Jun 2026 00:32:17 +0200 Subject: [PATCH 3/8] CU-86c2y3vu1-fix-sqlfluff-compilation-error --- .sqlfluff | 6 +++--- .sqlfluffignore | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 .sqlfluffignore diff --git a/.sqlfluff b/.sqlfluff index 52a5187..c1dd1ce 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -25,15 +25,15 @@ align_scope = bracketed # AS should be there in table aliases [sqlfluff:rules:aliasing.table] -aliasing.table = explicit +aliasing = explicit # AS should be there for renamed columns [sqlfluff:rules:aliasing.column] -aliasing.column = explicit +aliasing = explicit # AS should be there after SUM(), MAX() etc [sqlfluff:rules:aliasing.expression] -aliasing.expression = explicit +aliasing = explicit # use ALL numbers in GROUP BY, ORDER BY [sqlfluff:rules:ambiguous.column_references] diff --git a/.sqlfluffignore b/.sqlfluffignore new file mode 100644 index 0000000..6d2113b --- /dev/null +++ b/.sqlfluffignore @@ -0,0 +1,3 @@ +target +dbt_packages +logs From 43c9245771eb5e2b06f496b0e58f5bf8560b3d60 Mon Sep 17 00:00:00 2001 From: ateneva Date: Sat, 27 Jun 2026 00:48:14 +0200 Subject: [PATCH 4/8] CU-86c2y3vu1-ensure-sqlfluff-is-in-sync-with-sqlfluff-templater --- .github/workflows/linting-checks.yml | 2 +- .pre-commit-config.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linting-checks.yml b/.github/workflows/linting-checks.yml index 13ddc7a..b2d7bb5 100644 --- a/.github/workflows/linting-checks.yml +++ b/.github/workflows/linting-checks.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.11.14' cache: 'pip' - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e68320d..e9fca3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,9 +11,9 @@ repos: rev: 4.2.0 hooks: - id: sqlfluff-lint - additional_dependencies: ['dbt-bigquery==1.11.0', 'sqlfluff-templater-dbt==4.2.0'] + additional_dependencies: ['dbt-bigquery==1.11.0', 'sqlfluff==4.2.0', 'sqlfluff-templater-dbt==4.2.0'] - id: sqlfluff-fix - additional_dependencies: ['dbt-bigquery==1.11.0', 'sqlfluff-templater-dbt==4.2.0'] + additional_dependencies: ['dbt-bigquery==1.11.0', 'sqlfluff==4.2.0', 'sqlfluff-templater-dbt==4.2.0'] - repo: https://github.com/dbt-checkpoint/dbt-checkpoint rev: v1.2.1 From f9e4ffd1ea8cc27a4a15937a6012107c58a99e6f Mon Sep 17 00:00:00 2001 From: ateneva Date: Sat, 27 Jun 2026 01:00:04 +0200 Subject: [PATCH 5/8] CU-86c2y3vu1-explicityly-define-dbt-linting-dependencies --- .pre-commit-config.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9fca3b..3387c0b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,9 +11,18 @@ repos: rev: 4.2.0 hooks: - id: sqlfluff-lint - additional_dependencies: ['dbt-bigquery==1.11.0', 'sqlfluff==4.2.0', 'sqlfluff-templater-dbt==4.2.0'] + additional_dependencies: + - dbt-core==1.11.2 + - dbt-bigquery==1.11.0 + - sqlfluff==4.2.0 + - sqlfluff-templater-dbt==4.2.0 + - id: sqlfluff-fix - additional_dependencies: ['dbt-bigquery==1.11.0', 'sqlfluff==4.2.0', 'sqlfluff-templater-dbt==4.2.0'] + additional_dependencies: + - dbt-core==1.11.2 + - dbt-bigquery==1.11.0 + - sqlfluff==4.2.0 + - sqlfluff-templater-dbt==4.2.0 - repo: https://github.com/dbt-checkpoint/dbt-checkpoint rev: v1.2.1 From 716bec9a66357fbbd4f7e3a3c05ac78a6b988487 Mon Sep 17 00:00:00 2001 From: ateneva Date: Sat, 27 Jun 2026 01:14:14 +0200 Subject: [PATCH 6/8] CU-86c2y3vu1-dbt-deps-should-run-as-part-of-gha --- .github/workflows/linting-checks.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linting-checks.yml b/.github/workflows/linting-checks.yml index b2d7bb5..81fed10 100644 --- a/.github/workflows/linting-checks.yml +++ b/.github/workflows/linting-checks.yml @@ -9,7 +9,7 @@ permissions: pull-requests: write jobs: - pre-commit-checks: + linting-checks: runs-on: ubuntu-latest steps: @@ -29,6 +29,9 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + - name: Install dbt dependencies + run: dbt deps + - name: Set up Terraform uses: hashicorp/setup-terraform@v3 From fd706c9277184f11c4bd6e6831b935b867c48d5d Mon Sep 17 00:00:00 2001 From: ateneva Date: Sat, 27 Jun 2026 01:37:54 +0200 Subject: [PATCH 7/8] CU-86c2y3vu1-remove-data-validation-experiments --- dbt/analyses/compare_column_values.sql | 40 ------------------- dbt/analyses/quick_are_queries_identical.sql | 23 ----------- .../quick_are_relations_identical.sql | 20 ---------- 3 files changed, 83 deletions(-) delete mode 100644 dbt/analyses/compare_column_values.sql delete mode 100644 dbt/analyses/quick_are_queries_identical.sql delete mode 100644 dbt/analyses/quick_are_relations_identical.sql diff --git a/dbt/analyses/compare_column_values.sql b/dbt/analyses/compare_column_values.sql deleted file mode 100644 index 2a067a4..0000000 --- a/dbt/analyses/compare_column_values.sql +++ /dev/null @@ -1,40 +0,0 @@ -{{ - config( - materialized='table', - alias='compare_single_query_column' - ) -}} - - -{% set old_query %} - SELECT *, CONCAT(company_name, location) AS unique_identifier - FROM `data-geeking-gcp.the_data_challenge.data_engineer_jobs` -{% endset %} - -{% set new_query %} - SELECT *, CONCAT(company_name, location) AS unique_identifier - FROM `data-geeking-gcp.the_data_challenge.data_engineer_jobs_copy` -{% endset %} - -{{ audit_helper.compare_column_values( - a_query = old_query, - b_query = new_query, - primary_key = 'unique_identifier', - column_to_compare = 'job_title' - ) -}} - - -{# - [{ - "column_name": "job_title", - "match_status": "✅: perfect match", - "count_records": "2830", - "percent_of_total": "45.88" - }, { - "column_name": "job_title", - "match_status": "❌: ‍values do not match", - "count_records": "3338", - "percent_of_total": "54.12" - }] -#} diff --git a/dbt/analyses/quick_are_queries_identical.sql b/dbt/analyses/quick_are_queries_identical.sql deleted file mode 100644 index cfb5999..0000000 --- a/dbt/analyses/quick_are_queries_identical.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ - config( - materialized='table', - alias='compare_multiple_query_columns' - ) -}} - -{% set old_query %} - SELECT *, CONCAT(company_name, location) AS unique_identifier - FROM `data-geeking-gcp.the_data_challenge.data_engineer_jobs` -{% endset %} - -{% set new_query %} - SELECT *, CONCAT(company_name, location) AS unique_identifier - FROM `data-geeking-gcp.the_data_challenge.data_engineer_jobs_copy` -{% endset %} - -{{ audit_helper.quick_are_queries_identical( - query_a = old_query, - query_b = new_query, - columns = ['job_title', 'salary_estimate', 'job_description'] - ) -}} diff --git a/dbt/analyses/quick_are_relations_identical.sql b/dbt/analyses/quick_are_relations_identical.sql deleted file mode 100644 index 447fa42..0000000 --- a/dbt/analyses/quick_are_relations_identical.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ - config( - materialized='table', - alias='compare_multiple_table_columns' - ) -}} - -{% set old_table = adapter.get_relation( - database = "data-geeking-gcp", - schema = "the_data_challenge", - identifier = "data_engineer_jobs" -) -%} - -{% set new_table = "data-geeking-gcp.the_data_challenge.data_engineer_jobs_copy" %} -- noqa: LT05 - -{{ audit_helper.quick_are_relations_identical( - a_relation = old_table, - b_relation = new_table, - columns = ['job_title', 'salary_estimate', 'job_description'] -) }} From de9f2da7351c6648392f66dd29fe1b502a4d0923 Mon Sep 17 00:00:00 2001 From: ateneva Date: Sat, 27 Jun 2026 01:44:32 +0200 Subject: [PATCH 8/8] CU-86c2y3vu1-rename-step-in-github-workflow-for-more-clarity --- .github/workflows/linting-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting-checks.yml b/.github/workflows/linting-checks.yml index 81fed10..09020b3 100644 --- a/.github/workflows/linting-checks.yml +++ b/.github/workflows/linting-checks.yml @@ -24,7 +24,7 @@ jobs: python-version: '3.11.14' cache: 'pip' - - name: Install dependencies + - name: Install requirements run: | python -m pip install --upgrade pip pip install -r requirements.txt