From 477d6bef3ae1287c327cd9805d3fb1db6cbabe90 Mon Sep 17 00:00:00 2001 From: brendagutman Date: Tue, 2 Jun 2026 16:28:33 -0500 Subject: [PATCH 01/11] Check for permissions before running grant queries --- dbt_project/macros/grant_inc_devs_access.sql | 51 ++++++++++++++--- dbt_project/macros/grant_kf_devs_access.sql | 56 +++++++++++++++++++ .../macros/hold_generate_schema_name.sql | 35 ------------ 3 files changed, 100 insertions(+), 42 deletions(-) create mode 100644 dbt_project/macros/grant_kf_devs_access.sql delete mode 100644 dbt_project/macros/hold_generate_schema_name.sql diff --git a/dbt_project/macros/grant_inc_devs_access.sql b/dbt_project/macros/grant_inc_devs_access.sql index 26fd8b8..298c961 100644 --- a/dbt_project/macros/grant_inc_devs_access.sql +++ b/dbt_project/macros/grant_inc_devs_access.sql @@ -1,19 +1,56 @@ {% macro grant_inc_devs_access() %} {% if execute %} - {% set grantee = adapter.quote('include_users') %} - {% set run_schemas = schemas | unique if schemas is defined else [target.schema] %} + {% set grantee_name = 'include_users' %} + {% set grantee = adapter.quote(grantee_name) %} + {% set ns = namespace(run_schemas=[]) %} + + {% if results is defined %} + {% for res in results %} + {% set node_tags = res.node.tags if res.node.tags is defined else [] %} + {% if res.node.resource_type == 'model' and 'kids_first' not in node_tags %} + {% do ns.run_schemas.append(res.node.schema) %} + {% endif %} + {% endfor %} + {% endif %} + + {% set run_schemas = ns.run_schemas | unique %} + + {% if run_schemas | length == 0 %} + {% set run_schemas = schemas | unique if schemas is defined else [target.schema] %} + {% do log('No node results context found; using run schemas without tag-based exclusion', info=True) %} + {% endif %} + {% for schema_name in run_schemas %} {% set quoted_schema = adapter.quote(schema_name) %} + {% set schema_granted_sql %} + select has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') + {% endset %} + {% set schema_granted_result = run_query(schema_granted_sql) %} + {% set schema_granted = false %} - {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("grant select on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("grant usage, select on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% if schema_granted_result is not none and schema_granted_result.rows | length > 0 %} + {% set schema_granted = schema_granted_result.rows[0][0] %} + {% endif %} - {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant select on tables to " ~ grantee) %} - {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant usage, select on sequences to " ~ grantee) %} + {% if not schema_granted %} + {% do log('Applying include_users grants on schema ' ~ schema_name, info=True) %} + + {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant select, insert, update, delete on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant usage, select on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} + + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant select, insert, update, delete on tables to " ~ grantee) %} + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant usage, select on sequences to " ~ grantee) %} + {% else %} + {% do log('Skipping grants for schema ' ~ schema_name ~ ' (already granted)', info=True) %} + {% endif %} {% endfor %} {% endif %} {{ return('') }} +{% endmacro %} + +{% macro grant_devs_access() %} + {{ return(grant_inc_devs_access()) }} {% endmacro %} \ No newline at end of file diff --git a/dbt_project/macros/grant_kf_devs_access.sql b/dbt_project/macros/grant_kf_devs_access.sql new file mode 100644 index 0000000..947f2f0 --- /dev/null +++ b/dbt_project/macros/grant_kf_devs_access.sql @@ -0,0 +1,56 @@ +{% macro grant_inc_devs_access() %} + {% if execute %} + {% set grantee_name = 'kf_users' %} + {% set grantee = adapter.quote(grantee_name) %} + {% set ns = namespace(run_schemas=[]) %} + + {% if results is defined %} + {% for res in results %} + {% set node_tags = res.node.tags if res.node.tags is defined else [] %} + {% if res.node.resource_type == 'model' and 'include' not in node_tags %} + {% do ns.run_schemas.append(res.node.schema) %} + {% endif %} + {% endfor %} + {% endif %} + + {% set run_schemas = ns.run_schemas | unique %} + + {% if run_schemas | length == 0 %} + {% set run_schemas = schemas | unique if schemas is defined else [target.schema] %} + {% do log('No node results context found; using run schemas without tag-based exclusion', info=True) %} + {% endif %} + + + {% for schema_name in run_schemas %} + {% set quoted_schema = adapter.quote(schema_name) %} + {% set schema_granted_sql %} + select has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') + {% endset %} + {% set schema_granted_result = run_query(schema_granted_sql) %} + {% set schema_granted = false %} + + {% if schema_granted_result is not none and schema_granted_result.rows | length > 0 %} + {% set schema_granted = schema_granted_result.rows[0][0] %} + {% endif %} + + {% if not schema_granted %} + {% do log('Applying kf_users grants on schema ' ~ schema_name, info=True) %} + + {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant select, insert, update, delete on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant usage, select on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} + + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant select, insert, update, delete on tables to " ~ grantee) %} + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant usage, select on sequences to " ~ grantee) %} + {% else %} + {% do log('Skipping grants for schema ' ~ schema_name ~ ' (already granted)', info=True) %} + {% endif %} + {% endfor %} + {% endif %} + + {{ return('') }} +{% endmacro %} + +{% macro grant_devs_access() %} + {{ return(grant_kf_devs_access()) }} +{% endmacro %} \ No newline at end of file diff --git a/dbt_project/macros/hold_generate_schema_name.sql b/dbt_project/macros/hold_generate_schema_name.sql deleted file mode 100644 index 1a2fc2d..0000000 --- a/dbt_project/macros/hold_generate_schema_name.sql +++ /dev/null @@ -1,35 +0,0 @@ --- macros/generate_schema_name.sql -{% macro hold_generate_schema_name(custom_schema_name, node) -%} - - {%- set default_schema = target.schema -%} - - {# 1. Logic for Seed Files: result will be "dev_schema_import" #} - {%- if node.resource_type == 'seed' -%} - {%- if custom_schema_name is none -%} - {{ default_schema | trim }} - {%- else -%} - {{ default_schema | trim }}_{{ custom_schema_name | trim }} - {%- endif -%} - - {# 2. Unified Logic for Models #} - {%- else -%} - {# fqn_parts = [folder, subfolder, ..., model_name] #} - {%- set fqn_parts = node.fqn[1:-1] -%} - - {%- if fqn_parts | length == 1 -%} - {# Case: models/access/file.sql -> output: dev_schema_access #} - {{ default_schema }}_{{ fqn_parts[0] }} - - {%- elif fqn_parts | length > 1 -%} - {# Case: models/kf/study/src/files.sql -> output: dev_schema_kf_study #} - {# Exclude the last subfolder ('src') #} - {%- set schema_path = fqn_parts[:-1] | join('_') -%} - {{ default_schema }}_{{ schema_path }} - - {%- else -%} - {# Fallback for models in the root /models/ folder #} - {{ default_schema }} - {%- endif -%} - {%- endif -%} - -{%- endmacro %} From 634746552d390811eaddc14b413d66bd1d83f5cc Mon Sep 17 00:00:00 2001 From: brendagutman Date: Tue, 2 Jun 2026 16:31:04 -0500 Subject: [PATCH 02/11] Modify schema names and macro --- dbt_project/dbt_project.yml | 5 +-- dbt_project/macros/generate_schema_name.sql | 35 +++++++++++++++++++ .../models/include/brainpower/src/sources.yml | 2 +- 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 dbt_project/macros/generate_schema_name.sql diff --git a/dbt_project/dbt_project.yml b/dbt_project/dbt_project.yml index 5395215..ef96cca 100644 --- a/dbt_project/dbt_project.yml +++ b/dbt_project/dbt_project.yml @@ -37,7 +37,7 @@ models: +tags: - brainpower src: - +schema: inc_brainpower_src + +schema: inc_brainpower_src_bg int: +schema: inc_brainpower_int kids_first: @@ -68,4 +68,5 @@ models: - combined_stage +schema: combined on-run-end: - - "{{ grant_inc_devs_access() }}" \ No newline at end of file + - "{{ grant_inc_devs_access() }}" + - "{{ grant_kf_devs_access() }}" \ No newline at end of file diff --git a/dbt_project/macros/generate_schema_name.sql b/dbt_project/macros/generate_schema_name.sql new file mode 100644 index 0000000..a0c4595 --- /dev/null +++ b/dbt_project/macros/generate_schema_name.sql @@ -0,0 +1,35 @@ +-- macros/generate_schema_name.sql +{% macro generate_schema_name(custom_schema_name, node) -%} + + {%- set default_schema = target.schema -%} + + {# If an explicit schema is set in dbt_project.yml, use it as-is (no prefix) #} + {%- if custom_schema_name is not none -%} + {{ custom_schema_name | trim }} + + {# 1. Logic for Seed Files: result will be "dev_schema_import" #} + {%- elif node.resource_type == 'seed' -%} + {{ default_schema | trim }} + + {# 2. Unified Logic for Models #} + {%- else -%} + {# fqn_parts = [folder, subfolder, ..., model_name] #} + {%- set fqn_parts = node.fqn[1:-1] -%} + + {%- if fqn_parts | length == 1 -%} + {# Case: models/access/file.sql -> output: dev_schema_access #} + {{ default_schema }}_{{ fqn_parts[0] }} + + {%- elif fqn_parts | length > 1 -%} + {# Case: models/kf/study/src/files.sql -> output: dev_schema_kf_study #} + {# Exclude the last subfolder ('src') #} + {%- set schema_path = fqn_parts[:-1] | join('_') -%} + {{ default_schema }}_{{ schema_path }} + + {%- else -%} + {# Fallback for models in the root /models/ folder #} + {{ default_schema }} + {%- endif -%} + {%- endif -%} + +{%- endmacro %} diff --git a/dbt_project/models/include/brainpower/src/sources.yml b/dbt_project/models/include/brainpower/src/sources.yml index fcb5b86..52b9f62 100644 --- a/dbt_project/models/include/brainpower/src/sources.yml +++ b/dbt_project/models/include/brainpower/src/sources.yml @@ -1,6 +1,6 @@ sources: - name: brainpower - schema: inc_brainpower_src + schema: inc_brainpower_raw tables: - name: bp_age_event_latency description: Source table for bp_age_event_latency. From efff79cc5e9616d4b72bc048f09770f18cd9e172 Mon Sep 17 00:00:00 2001 From: brendagutman Date: Wed, 3 Jun 2026 16:40:27 -0500 Subject: [PATCH 03/11] Minor modifications --- dbt_project/dbt_project.yml | 2 +- dbt_project/macros/grant_inc_devs_access.sql | 4 ---- dbt_project/macros/grant_kf_devs_access.sql | 6 +----- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/dbt_project/dbt_project.yml b/dbt_project/dbt_project.yml index ef96cca..2c5fbf7 100644 --- a/dbt_project/dbt_project.yml +++ b/dbt_project/dbt_project.yml @@ -37,7 +37,7 @@ models: +tags: - brainpower src: - +schema: inc_brainpower_src_bg + +schema: inc_brainpower_src int: +schema: inc_brainpower_int kids_first: diff --git a/dbt_project/macros/grant_inc_devs_access.sql b/dbt_project/macros/grant_inc_devs_access.sql index 298c961..ada1e83 100644 --- a/dbt_project/macros/grant_inc_devs_access.sql +++ b/dbt_project/macros/grant_inc_devs_access.sql @@ -49,8 +49,4 @@ {% endif %} {{ return('') }} -{% endmacro %} - -{% macro grant_devs_access() %} - {{ return(grant_inc_devs_access()) }} {% endmacro %} \ No newline at end of file diff --git a/dbt_project/macros/grant_kf_devs_access.sql b/dbt_project/macros/grant_kf_devs_access.sql index 947f2f0..f5ad94c 100644 --- a/dbt_project/macros/grant_kf_devs_access.sql +++ b/dbt_project/macros/grant_kf_devs_access.sql @@ -1,4 +1,4 @@ -{% macro grant_inc_devs_access() %} +{% macro grant_kf_devs_access() %} {% if execute %} {% set grantee_name = 'kf_users' %} {% set grantee = adapter.quote(grantee_name) %} @@ -49,8 +49,4 @@ {% endif %} {{ return('') }} -{% endmacro %} - -{% macro grant_devs_access() %} - {{ return(grant_kf_devs_access()) }} {% endmacro %} \ No newline at end of file From 66b52ce024115b8b2cdaae8a82e84d566b7b0b6c Mon Sep 17 00:00:00 2001 From: brendagutman Date: Wed, 3 Jun 2026 17:01:54 -0500 Subject: [PATCH 04/11] Revert generate_schema_name --- dbt_project/macros/generate_schema_name.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dbt_project/macros/generate_schema_name.sql b/dbt_project/macros/generate_schema_name.sql index a0c4595..3272dea 100644 --- a/dbt_project/macros/generate_schema_name.sql +++ b/dbt_project/macros/generate_schema_name.sql @@ -3,13 +3,13 @@ {%- set default_schema = target.schema -%} - {# If an explicit schema is set in dbt_project.yml, use it as-is (no prefix) #} - {%- if custom_schema_name is not none -%} - {{ custom_schema_name | trim }} - {# 1. Logic for Seed Files: result will be "dev_schema_import" #} - {%- elif node.resource_type == 'seed' -%} - {{ default_schema | trim }} + {%- if node.resource_type == 'seed' -%} + {%- if custom_schema_name is none -%} + {{ default_schema | trim }} + {%- else -%} + {{ default_schema | trim }}_{{ custom_schema_name | trim }} + {%- endif -%} {# 2. Unified Logic for Models #} {%- else -%} From 9630c2a8d90f3421ce274df0a651c91536845855 Mon Sep 17 00:00:00 2001 From: brendagutman Date: Thu, 4 Jun 2026 11:12:54 -0500 Subject: [PATCH 05/11] Add the adr and cross-project macro --- dbt_project/dbt_project.yml | 4 +- dbt_project/macros/_macros.yml | 123 ++++++++++++++++++ dbt_project/macros/grant_kf_devs_access.sql | 52 -------- .../grant_devs_access.sql} | 30 +++-- .../macros/grants/grant_inc_devs_access.sql | 3 + .../macros/grants/grant_kf_devs_access.sql | 3 + .../grants/grant_schema_role_access.sql | 41 ++++++ .../adr-012-granting-access-to-roles | 87 +++++++++++++ 8 files changed, 280 insertions(+), 63 deletions(-) delete mode 100644 dbt_project/macros/grant_kf_devs_access.sql rename dbt_project/macros/{grant_inc_devs_access.sql => grants/grant_devs_access.sql} (63%) create mode 100644 dbt_project/macros/grants/grant_inc_devs_access.sql create mode 100644 dbt_project/macros/grants/grant_kf_devs_access.sql create mode 100644 dbt_project/macros/grants/grant_schema_role_access.sql create mode 100644 docs/arch/dbt_repo_design/adr-012-granting-access-to-roles diff --git a/dbt_project/dbt_project.yml b/dbt_project/dbt_project.yml index 2c5fbf7..1d7f766 100644 --- a/dbt_project/dbt_project.yml +++ b/dbt_project/dbt_project.yml @@ -68,5 +68,5 @@ models: - combined_stage +schema: combined on-run-end: - - "{{ grant_inc_devs_access() }}" - - "{{ grant_kf_devs_access() }}" \ No newline at end of file + - "{{ grant_devs_access(tag='include', users_role='include_users') }}" + - "{{ grant_devs_access(tag='kids_first', users_role='kf_users') }}" \ No newline at end of file diff --git a/dbt_project/macros/_macros.yml b/dbt_project/macros/_macros.yml index e69de29..d5daaf8 100644 --- a/dbt_project/macros/_macros.yml +++ b/dbt_project/macros/_macros.yml @@ -0,0 +1,123 @@ +version: 2 + +macros: + - name: grant_devs_access + description: > + Shared on-run-end helper that grants a role full access to every schema + produced by models matching a given project tag during the current dbt run. + The two project-specific wrappers (`grant_inc_devs_access`, + `grant_kf_devs_access`) delegate directly to this macro with their + fixed `tag` / `users_role` values. + + + **Behavior:** + + 1. Validates that both `tag` and `users_role` are provided; raises a + compiler error if either is missing. + + 2. Iterates over `results` and collects the schema of every model whose + tags contain `tag` — ensuring only schemas belonging to the + specified project are touched. + + 3. If no matching models are found (e.g. the run contained no models with + that tag), logs an INFO message and exits without issuing any SQL. + + 4. For each collected schema, checks `has_schema_privilege(..., 'USAGE')`. + Already-granted schemas are skipped with an INFO log. + + 5. For schemas that still need granting, issues: + - `GRANT USAGE ON SCHEMA` — allows the role to see and interact with the schema. + - `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA` — full DML + access on every table that currently exists in the schema. + - `GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA` — access to all existing + sequences (required for auto-increment / serial columns). + - `ALTER DEFAULT PRIVILEGES ... GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES` + — ensures future tables created in this schema are automatically accessible. + - `ALTER DEFAULT PRIVILEGES ... GRANT USAGE, SELECT ON SEQUENCES` — same + future coverage for sequences. + + + 6. All actions and skips are logged at `INFO` level. + + arguments: + - name: tag + type: string + description: > + The dbt tag used to identify models belonging to the target project. + For example, `include` or `kids_first`. Only models whose tags + contain this value will have their schemas granted. + + - name: users_role + type: string + description: > + The database role to receive the grants. Must be an existing role + in the target database (e.g. `include_users`, `kf_users`). + + - name: grant_inc_devs_access + description: > + On-run-end wrapper around `grant_devs_access`. Grants `include_users` + access to all schemas produced by models tagged `include` in the current + run (e.g. `brainpower`, `aadsc` src/int schemas). Delegates entirely to + `grant_devs_access(tag='include', users_role='include_users')`. + + - name: grant_kf_devs_access + description: > + On-run-end wrapper around `grant_devs_access`. Grants `kf_users` + access to all schemas produced by models tagged `kids_first` in the + current run. Delegates entirely to + `grant_devs_access(tag='kids_first', users_role='kf_users')`. + + - name: grant_schema_role_access + description: > + Standalone utility macro that grants a specified role full access to a target + schema. Unlike the on-run-end grant macros (`grant_inc_devs_access`, + `grant_kf_devs_access`), this macro is not tied to a dbt run lifecycle and is + intended to be called manually via `dbt run-operation` whenever a new role needs + access to the `combined`, `access`, or `*_export` schemas. + + + **Behavior:** + + 1. Validates that both `target_schema` and `users_role` are provided; raises a + compiler error if either is missing. + + 2. Checks whether the role already holds `USAGE` privilege on the schema using + `has_schema_privilege`. If access is already granted, the macro logs a skip + message and exits without issuing any SQL — making it safe to run repeatedly + without accumulating duplicate grants. + + 3. If access is not yet granted, the macro issues the following statements + against the target schema: + - `GRANT USAGE ON SCHEMA` — allows the role to see and interact with the schema. + - `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA` — full DML + access on every table that currently exists in the schema. + - `GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA` — access to all existing + sequences (required for auto-increment / serial columns). + - `ALTER DEFAULT PRIVILEGES ... GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES` + — ensures future tables created in this schema are automatically accessible. + - `ALTER DEFAULT PRIVILEGES ... GRANT USAGE, SELECT ON SEQUENCES` — same + future coverage for sequences. + + 4. All actions and skips are logged to the dbt console at `INFO` level. + + + **Example usage:** + + ```bash + dbt run-operation grant_schema_role_access \ + --args '{"target_schema": "combined", "users_role": "analyst_role"}' + ``` + + arguments: + - name: target_schema + type: string + description: > + The name of the database schema to grant access to. Intended for shared, + non-program-specific schemas such as `combined`, `access`, or `fhir_export`. + + + - name: users_role + type: string + description: > + The name of the database role to receive the grants. This shouldbe + an existing role in the target database. diff --git a/dbt_project/macros/grant_kf_devs_access.sql b/dbt_project/macros/grant_kf_devs_access.sql deleted file mode 100644 index f5ad94c..0000000 --- a/dbt_project/macros/grant_kf_devs_access.sql +++ /dev/null @@ -1,52 +0,0 @@ -{% macro grant_kf_devs_access() %} - {% if execute %} - {% set grantee_name = 'kf_users' %} - {% set grantee = adapter.quote(grantee_name) %} - {% set ns = namespace(run_schemas=[]) %} - - {% if results is defined %} - {% for res in results %} - {% set node_tags = res.node.tags if res.node.tags is defined else [] %} - {% if res.node.resource_type == 'model' and 'include' not in node_tags %} - {% do ns.run_schemas.append(res.node.schema) %} - {% endif %} - {% endfor %} - {% endif %} - - {% set run_schemas = ns.run_schemas | unique %} - - {% if run_schemas | length == 0 %} - {% set run_schemas = schemas | unique if schemas is defined else [target.schema] %} - {% do log('No node results context found; using run schemas without tag-based exclusion', info=True) %} - {% endif %} - - - {% for schema_name in run_schemas %} - {% set quoted_schema = adapter.quote(schema_name) %} - {% set schema_granted_sql %} - select has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') - {% endset %} - {% set schema_granted_result = run_query(schema_granted_sql) %} - {% set schema_granted = false %} - - {% if schema_granted_result is not none and schema_granted_result.rows | length > 0 %} - {% set schema_granted = schema_granted_result.rows[0][0] %} - {% endif %} - - {% if not schema_granted %} - {% do log('Applying kf_users grants on schema ' ~ schema_name, info=True) %} - - {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("grant select, insert, update, delete on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("grant usage, select on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} - - {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant select, insert, update, delete on tables to " ~ grantee) %} - {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant usage, select on sequences to " ~ grantee) %} - {% else %} - {% do log('Skipping grants for schema ' ~ schema_name ~ ' (already granted)', info=True) %} - {% endif %} - {% endfor %} - {% endif %} - - {{ return('') }} -{% endmacro %} \ No newline at end of file diff --git a/dbt_project/macros/grant_inc_devs_access.sql b/dbt_project/macros/grants/grant_devs_access.sql similarity index 63% rename from dbt_project/macros/grant_inc_devs_access.sql rename to dbt_project/macros/grants/grant_devs_access.sql index ada1e83..fd95b16 100644 --- a/dbt_project/macros/grant_inc_devs_access.sql +++ b/dbt_project/macros/grants/grant_devs_access.sql @@ -1,25 +1,37 @@ -{% macro grant_inc_devs_access() %} +{% macro grant_devs_access(tag, users_role) %} {% if execute %} - {% set grantee_name = 'include_users' %} + {% set command_name = flags.WHICH if flags is defined and flags.WHICH is defined else none %} + {% if command_name not in ['run', 'build'] %} + {% do log('grant_devs_access: skipping for command ' ~ (command_name or 'unknown') ~ '; grants are only applied for run/build', info=True) %} + {{ return('') }} + {% endif %} + + {% if not tag %} + {% do exceptions.raise_compiler_error('grant_devs_access: tag is required') %} + {% endif %} + + {% if not users_role %} + {% do exceptions.raise_compiler_error('grant_devs_access: users_role is required') %} + {% endif %} + + {% set grantee_name = users_role %} {% set grantee = adapter.quote(grantee_name) %} {% set ns = namespace(run_schemas=[]) %} {% if results is defined %} {% for res in results %} {% set node_tags = res.node.tags if res.node.tags is defined else [] %} - {% if res.node.resource_type == 'model' and 'kids_first' not in node_tags %} + {% if res.node.resource_type == 'model' and tag in node_tags %} {% do ns.run_schemas.append(res.node.schema) %} {% endif %} {% endfor %} {% endif %} - {% set run_schemas = ns.run_schemas | unique %} + {% set run_schemas = ns.run_schemas | unique | list %} {% if run_schemas | length == 0 %} - {% set run_schemas = schemas | unique if schemas is defined else [target.schema] %} - {% do log('No node results context found; using run schemas without tag-based exclusion', info=True) %} + {% do log('grant_devs_access: no models with the ' ~ tag ~ ' tag found in run results; skipping grants', info=True) %} {% endif %} - {% for schema_name in run_schemas %} {% set quoted_schema = adapter.quote(schema_name) %} @@ -34,7 +46,7 @@ {% endif %} {% if not schema_granted %} - {% do log('Applying include_users grants on schema ' ~ schema_name, info=True) %} + {% do log('Applying ' ~ grantee_name ~ ' grants on schema ' ~ schema_name, info=True) %} {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} {% do run_query("grant select, insert, update, delete on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} @@ -49,4 +61,4 @@ {% endif %} {{ return('') }} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/dbt_project/macros/grants/grant_inc_devs_access.sql b/dbt_project/macros/grants/grant_inc_devs_access.sql new file mode 100644 index 0000000..f330cc2 --- /dev/null +++ b/dbt_project/macros/grants/grant_inc_devs_access.sql @@ -0,0 +1,3 @@ +{% macro grant_inc_devs_access() %} + {{ grant_devs_access(tag='include', users_role='include_users') }} +{% endmacro %} \ No newline at end of file diff --git a/dbt_project/macros/grants/grant_kf_devs_access.sql b/dbt_project/macros/grants/grant_kf_devs_access.sql new file mode 100644 index 0000000..024b9d3 --- /dev/null +++ b/dbt_project/macros/grants/grant_kf_devs_access.sql @@ -0,0 +1,3 @@ +{% macro grant_kf_devs_access() %} + {{ grant_devs_access(tag='kids_first', users_role='kf_users') }} +{% endmacro %} \ No newline at end of file diff --git a/dbt_project/macros/grants/grant_schema_role_access.sql b/dbt_project/macros/grants/grant_schema_role_access.sql new file mode 100644 index 0000000..03a1cd3 --- /dev/null +++ b/dbt_project/macros/grants/grant_schema_role_access.sql @@ -0,0 +1,41 @@ +{% macro grant_schema_role_access(target_schema, users_role) %} + {% if execute %} + {% if not target_schema %} + {% do exceptions.raise_compiler_error('grant_schema_role_access: target_schema is required') %} + {% endif %} + + {% if not users_role %} + {% do exceptions.raise_compiler_error('grant_schema_role_access: users_role is required') %} + {% endif %} + + {% set schema_name = target_schema %} + {% set grantee_name = users_role %} + {% set quoted_schema = adapter.quote(schema_name) %} + {% set grantee = adapter.quote(grantee_name) %} + + {% set schema_granted_sql %} + select has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') + {% endset %} + {% set schema_granted_result = run_query(schema_granted_sql) %} + {% set schema_granted = false %} + + {% if schema_granted_result is not none and schema_granted_result.rows | length > 0 %} + {% set schema_granted = schema_granted_result.rows[0][0] %} + {% endif %} + + {% if not schema_granted %} + {% do log('Applying ' ~ grantee_name ~ ' grants on schema ' ~ schema_name, info=True) %} + + {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant select, insert, update, delete on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant usage, select on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} + + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant select, insert, update, delete on tables to " ~ grantee) %} + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant usage, select on sequences to " ~ grantee) %} + {% else %} + {% do log('Skipping grants for schema ' ~ schema_name ~ ' (already granted for role ' ~ grantee_name ~ ')', info=True) %} + {% endif %} + {% endif %} + + {{ return('') }} +{% endmacro %} diff --git a/docs/arch/dbt_repo_design/adr-012-granting-access-to-roles b/docs/arch/dbt_repo_design/adr-012-granting-access-to-roles new file mode 100644 index 0000000..ad07563 --- /dev/null +++ b/docs/arch/dbt_repo_design/adr-012-granting-access-to-roles @@ -0,0 +1,87 @@ +--- +# These are optional metadata elements. Feel free to remove any of them. +status: proposed +date: 2026-06-04 +--- + +# Granting Access To Roles In The dbt Pipeline + +## Context + +The warehouse role model is defined in +[ADR-008](../warehouse-administration/adr-008-warehouse-roles.md), which +establishes that permissions are granted to roles and users inherit access via +role membership. + +Within this dbt repository, model schemas are created across multiple +project-specific and shared layers: + +1. Project-specific study layers, currently `include` and `kids_first`. +2. Shared layers, such as `access`, `export`, and `combined`. + +Grant logic must satisfy two operational needs: + +1. Automatically apply role access to project-specific schemas whenever new study + models are created in normal pipeline runs. +2. Apply grants to shared schemas only when needed, since shared schemas are + not tied to one project and role onboarding is less frequent. + +## Decision + +Access grants are managed through dbt macros with two execution patterns: + +1. **Automatic project-specific grants at pipeline end**: + `on-run-end` invokes `grant_devs_access(tag, users_role)` for each project role. + The macro scopes schemas by model tag so each project role only receives access + to schemas generated by that project's models. + +2. **Manual shared-schema grants**: + `grant_schema_role_access(target_schema, users_role)` is run intentionally + with `dbt run-operation` when a new role should receive access to shared + schemas (for example: `combined`, `access`, `fhir_export`). + +To avoid misleading behavior during non-execution commands, project-specific grant +logic applies only for `dbt run` and `dbt build` execution contexts. + +## Why This Approach + +This approach was chosen because it balances automation with explicit control: + +1. It keeps project-role grants continuous and low-maintenance during regular model + delivery. +2. It prevents cross-project over-granting by requiring tag-based schema matching. +3. It keeps shared-schema grants explicit and auditable when new roles are + onboarded. +4. It aligns with role-based access principles defined in ADR-008. + +## Alternatives Considered + +1. **Grant all schemas to both project roles automatically**: + rejected because it violates project boundary expectations and creates + unnecessary privilege overlap. + +2. **Manual grants only for all schemas**: + rejected because it is operationally heavy and error-prone for frequent + project-specific schema changes. + +3. **Separate duplicated macros per project with independent logic**: + rejected in favor of a shared implementation (`grant_devs_access`) to reduce + drift and maintenance overhead. + +## Consequences + +1. `include_users` and `kf_users` grants are applied automatically only for + schemas produced by tagged models in their own project domain. +2. Shared schemas require intentional manual grant operations when adding new + roles. +3. Pipeline behavior is clearer: compile-only workflows do not attempt to apply + grants. + +## Implementation Notes + +Detailed macro descriptions, arguments, and examples are maintained in: + +1. [dbt_project/macros/_macros.yml](../../../dbt_project/macros/_macros.yml) + +This ADR documents design intent and operating logic; `_macros.yml` remains the +source of truth for macro-level usage documentation. From 556fe0f3b013a1a60288f77956a14e331c5b6a1e Mon Sep 17 00:00:00 2001 From: brendagutman Date: Thu, 4 Jun 2026 11:34:53 -0500 Subject: [PATCH 06/11] Modify to grant dev access to dev schemas only --- dbt_project/macros/grants/grant_devs_access.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_project/macros/grants/grant_devs_access.sql b/dbt_project/macros/grants/grant_devs_access.sql index fd95b16..5e6172d 100644 --- a/dbt_project/macros/grants/grant_devs_access.sql +++ b/dbt_project/macros/grants/grant_devs_access.sql @@ -21,7 +21,7 @@ {% if results is defined %} {% for res in results %} {% set node_tags = res.node.tags if res.node.tags is defined else [] %} - {% if res.node.resource_type == 'model' and tag in node_tags %} + {% if res.node.resource_type == 'model' and tag in node_tags and 'dev_' in (res.node.schema | lower) %} {% do ns.run_schemas.append(res.node.schema) %} {% endif %} {% endfor %} @@ -30,7 +30,7 @@ {% set run_schemas = ns.run_schemas | unique | list %} {% if run_schemas | length == 0 %} - {% do log('grant_devs_access: no models with the ' ~ tag ~ ' tag found in run results; skipping grants', info=True) %} + {% do log('grant_devs_access: no models with the ' ~ tag ~ ' tag and schemas containing dev_ found in run results; skipping grants', info=True) %} {% endif %} {% for schema_name in run_schemas %} From 3c65c845f971b808b4f9fcb951eae5f2b4d03dcb Mon Sep 17 00:00:00 2001 From: brendagutman Date: Thu, 4 Jun 2026 15:41:20 -0500 Subject: [PATCH 07/11] Remove unrelated changes --- dbt_project/macros/generate_schema_name.sql | 35 ------------------- .../models/include/brainpower/src/sources.yml | 2 +- 2 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 dbt_project/macros/generate_schema_name.sql diff --git a/dbt_project/macros/generate_schema_name.sql b/dbt_project/macros/generate_schema_name.sql deleted file mode 100644 index 3272dea..0000000 --- a/dbt_project/macros/generate_schema_name.sql +++ /dev/null @@ -1,35 +0,0 @@ --- macros/generate_schema_name.sql -{% macro generate_schema_name(custom_schema_name, node) -%} - - {%- set default_schema = target.schema -%} - - {# 1. Logic for Seed Files: result will be "dev_schema_import" #} - {%- if node.resource_type == 'seed' -%} - {%- if custom_schema_name is none -%} - {{ default_schema | trim }} - {%- else -%} - {{ default_schema | trim }}_{{ custom_schema_name | trim }} - {%- endif -%} - - {# 2. Unified Logic for Models #} - {%- else -%} - {# fqn_parts = [folder, subfolder, ..., model_name] #} - {%- set fqn_parts = node.fqn[1:-1] -%} - - {%- if fqn_parts | length == 1 -%} - {# Case: models/access/file.sql -> output: dev_schema_access #} - {{ default_schema }}_{{ fqn_parts[0] }} - - {%- elif fqn_parts | length > 1 -%} - {# Case: models/kf/study/src/files.sql -> output: dev_schema_kf_study #} - {# Exclude the last subfolder ('src') #} - {%- set schema_path = fqn_parts[:-1] | join('_') -%} - {{ default_schema }}_{{ schema_path }} - - {%- else -%} - {# Fallback for models in the root /models/ folder #} - {{ default_schema }} - {%- endif -%} - {%- endif -%} - -{%- endmacro %} diff --git a/dbt_project/models/include/brainpower/src/sources.yml b/dbt_project/models/include/brainpower/src/sources.yml index 52b9f62..fcb5b86 100644 --- a/dbt_project/models/include/brainpower/src/sources.yml +++ b/dbt_project/models/include/brainpower/src/sources.yml @@ -1,6 +1,6 @@ sources: - name: brainpower - schema: inc_brainpower_raw + schema: inc_brainpower_src tables: - name: bp_age_event_latency description: Source table for bp_age_event_latency. From e39cf7c02104772d1193d0ae4ac566a0f56963bc Mon Sep 17 00:00:00 2001 From: brendagutman Date: Thu, 4 Jun 2026 15:51:08 -0500 Subject: [PATCH 08/11] Change filetype of the adr to md --- ...anting-access-to-roles => adr-012-granting-access-to-roles.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/arch/dbt_repo_design/{adr-012-granting-access-to-roles => adr-012-granting-access-to-roles.md} (100%) diff --git a/docs/arch/dbt_repo_design/adr-012-granting-access-to-roles b/docs/arch/dbt_repo_design/adr-012-granting-access-to-roles.md similarity index 100% rename from docs/arch/dbt_repo_design/adr-012-granting-access-to-roles rename to docs/arch/dbt_repo_design/adr-012-granting-access-to-roles.md From 67b3955bb4c0d39cb0a908df3df0d0eaf29d7cdc Mon Sep 17 00:00:00 2001 From: brendagutman Date: Thu, 23 Jul 2026 14:17:16 -0500 Subject: [PATCH 09/11] Modify to use on all to grant permission --- dbt_project/macros/_macros.yml | 493 ++++++++++++++++++ .../macros/grants/grant_devs_access.sql | 16 +- .../grants/grant_schema_role_access.sql | 16 +- .../macros/pipeline/combined_union.sql | 30 +- 4 files changed, 532 insertions(+), 23 deletions(-) diff --git a/dbt_project/macros/_macros.yml b/dbt_project/macros/_macros.yml index e69de29..21ffcc0 100644 --- a/dbt_project/macros/_macros.yml +++ b/dbt_project/macros/_macros.yml @@ -0,0 +1,493 @@ +version: 2 + +macros: + - name: generate_placeholder_global_ids_table + description: | + Creates an empty placeholder global_ids table with the expected schema. + Used when a downstream pipeline expects the table to exist before real + data is populated. + + Required args: + - schema_name + Optional args: + - table_name (defaults to global_ids) + + Command line usage: + dbt run-operation generate_placeholder_global_ids_table --args '{"schema_name": "inc_brainpower_raw"}' + + Explicit table name: + dbt run-operation generate_placeholder_global_ids_table --args '{"schema_name": "inc_brainpower_raw", "table_name": "global_ids"}' + + - name: store_profile + description: | + Profiles a relation and stores results in a table. + This is the primary ad hoc entry point and supports append-only history: + only changed profile rows are inserted with a new profiled_at timestamp. + + Required args: + - profile_table + + Optional args: + - profile_database (defaults to target.database) + - profile_schema (defaults to target.schema) + - profile_include_columns (list) + - profile_exclude_columns (list) + - profile_where_clause + - profile_group_by (list) + - adhoc_database (defaults to profile_database) + - adhoc_schema (defaults to profile_schema) + - adhoc_table (defaults to profile_) + + Command line usage: + dbt run-operation store_profile --args '{"profile_schema": "dev_inc_brainpower_src", "profile_table": "inc_brainpower_src_datasets"}' + + - name: get_profile + description: | + Backward-compatible wrapper around store_profile. + Accepts an args dictionary and forwards all supported profile options. + + Required args: + - args (dictionary) + + Command line usage: + dbt run-operation get_profile --args '{"profile_schema": "dev_inc_brainpower_src", "profile_table": "inc_brainpower_src_datasets"}' + + - name: run_get_profile + description: | + Backward-compatible wrapper around store_profile. + Accepts an args dictionary and forwards all supported profile options. + + Required args: + - args (dictionary) + + Command line usage: + dbt run-operation run_get_profile --args '{"profile_schema": "dev_inc_brainpower_src", "profile_table": "inc_brainpower_src_datasets"}' + + - name: combined_stb_relations + description: | + Builds a list of ref() relations for a combined union model. + Used by union-style models to collect study-specific stb tables. + + Quickstart: + macros/pipeline/docs/combined_stb_relations_quickstart.md + + Required args: + - table_name + Optional args: + - studies_var (defaults to combined_studies) + + Used from model SQL, for example: + combined_stb_relations(table_name='patient') + + - name: combined_union_from_current_model + description: | + Unions the study-specific relations that correspond to the current + combined_ model. Intended for use inside combined union model SQL. + + Quickstart: + macros/pipeline/docs/combined_union_from_current_model_quickstart.md + + Optional args: + - studies_var (defaults to combined_studies) + + Used from model SQL, for example: + combined_union_from_current_model() + + - name: generate_descriptor_view_sql + description: | + Generates placeholder SQL for a descriptor view based on program and + resource exceptions. + Used by descriptor-view models that need a compile-safe SQL stub. + + Quickstart: + macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md + + Required args: + - program_id + - dewrangle_study_id + Optional args: + - inclusion_resources (list) + - exclusion_resources (list) + + Used from model SQL, for example: + generate_descriptor_view_sql(program_id='inc', dewrangle_study_id='brainpower', exclusion_resources=[]) + + - name: generate_schema_name + description: | + Custom schema naming macro used by dbt for seeds and models. + It prefixes the target schema and appends the model folder path so dev + runs land in predictable schema names. + + Quickstart: + macros/pipeline/docs/generate_schema_name_quickstart.md + + Used automatically by dbt; not called directly in model SQL. + + - name: generate_stb_sql + description: | + Generates SQL that joins descriptor sources to GID lookup rows. + Used by STB-style models to map descriptors to identifiers. + + Quickstart: + macros/pipeline/docs/generate_stb_sql_quickstart.md + + Required args: + - base_source + Optional args: + - gid_lookup + - descriptor_sources (list) + + Used from model SQL, for example: + generate_stb_sql(base_source='my_base_model', gid_lookup='my_gid_lookup', descriptor_sources=[{'Patient': {} }]) + + - name: normalize_descriptors + description: | + Returns a normalized concatenated SQL expression for one or more + descriptor columns. + Used when building stable composite descriptors from multiple fields. + + Quickstart: + macros/pipeline/docs/normalize_descriptors_quickstart.md + + Default behavior per descriptor entry: + - trims leading and trailing whitespace + - collapses internal whitespace to a single space + - preserves case + - preserves special characters + - escapes the chosen delimiter + - converts blank strings to null + + Keep the common case simple by passing a list of SQL expressions: + normalize_descriptors(descriptor_cols=['id', 'study_code']) + + For overrides, pass a list item as a dictionary with expr plus only the + options you need to change: + normalize_descriptors( + descriptor_cols=[ + 'id', + {'expr': 'file_name', 'escape_delimiter': false}, + {'expr': 'normalized_hpo_term', 'keep_case': false} + ], + delimiter=';' + ) + + A fixed descriptor string can also be normalized with its own override + options: + normalize_descriptors( + descriptor_str='brainpower,v2', + descriptor_str_options={'escape_delimiter': true, 'keep_case': true} + ) + + Required args: + - descriptor_cols (list) + Optional args: + - descriptor_str + - delimiter (defaults to ,) + - default_options (dictionary) + - descriptor_str_options (dictionary) + + - name: grant_devs_access + description: > + Shared on-run-end helper that grants a role full access to every schema + produced by models matching a given project tag during the current dbt run. + The two project-specific wrappers (`grant_inc_devs_access`, + `grant_kf_devs_access`) delegate directly to this macro with their + fixed `tag` / `users_role` values. + + + **Behavior:** + + 1. Validates that both `tag` and `users_role` are provided; raises a + compiler error if either is missing. + + 2. Iterates over `results` and collects the schema of every model whose + tags contain `tag` — ensuring only schemas belonging to the + specified project are touched. + + 3. If no matching models are found (e.g. the run contained no models with + that tag), logs an INFO message and exits without issuing any SQL. + + 4. For each collected schema, checks `has_schema_privilege(..., 'USAGE')`. + Already-granted schemas are skipped with an INFO log. + + 5. For schemas that still need granting, issues: + - `GRANT USAGE ON SCHEMA` — allows the role to see and interact with the schema. + - `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA` — full DML + access on every table that currently exists in the schema. + - `GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA` — access to all existing + sequences (required for auto-increment / serial columns). + - `ALTER DEFAULT PRIVILEGES ... GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES` + — ensures future tables created in this schema are automatically accessible. + - `ALTER DEFAULT PRIVILEGES ... GRANT USAGE, SELECT ON SEQUENCES` — same + future coverage for sequences. + + + 6. All actions and skips are logged at `INFO` level. + + arguments: + - name: tag + type: string + description: > + The dbt tag used to identify models belonging to the target project. + For example, `include` or `kids_first`. Only models whose tags + contain this value will have their schemas granted. + + - name: users_role + type: string + description: > + The database role to receive the grants. Must be an existing role + in the target database (e.g. `include_users`, `kf_users`). + + - name: grant_inc_devs_access + description: > + On-run-end wrapper around `grant_devs_access`. Grants `include_users` + access to all schemas produced by models tagged `include` in the current + run (e.g. `brainpower`, `aadsc` src/int schemas). Delegates entirely to + `grant_devs_access(tag='include', users_role='include_users')`. + + - name: grant_kf_devs_access + description: > + On-run-end wrapper around `grant_devs_access`. Grants `kf_users` + access to all schemas produced by models tagged `kids_first` in the + current run. Delegates entirely to + `grant_devs_access(tag='kids_first', users_role='kf_users')`. + + - name: grant_schema_role_access + description: > + Standalone utility macro that grants a specified role full access to a target + schema. Unlike the on-run-end grant macros (`grant_inc_devs_access`, + `grant_kf_devs_access`), this macro is not tied to a dbt run lifecycle and is + intended to be called manually via `dbt run-operation` whenever a new role needs + access to the `combined`, `access`, or `*_export` schemas. + + + **Behavior:** + + 1. Validates that both `target_schema` and `users_role` are provided; raises a + compiler error if either is missing. + + 2. Checks whether the role already holds `USAGE` privilege on the schema using + `has_schema_privilege`. If access is already granted, the macro logs a skip + message and exits without issuing any SQL — making it safe to run repeatedly + without accumulating duplicate grants. + + 3. If access is not yet granted, the macro issues the following statements + against the target schema: + - `GRANT USAGE ON SCHEMA` — allows the role to see and interact with the schema. + - `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA` — full DML + access on every table that currently exists in the schema. + - `GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA` — access to all existing + sequences (required for auto-increment / serial columns). + - `ALTER DEFAULT PRIVILEGES ... GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES` + — ensures future tables created in this schema are automatically accessible. + - `ALTER DEFAULT PRIVILEGES ... GRANT USAGE, SELECT ON SEQUENCES` — same + future coverage for sequences. + + 4. All actions and skips are logged to the dbt console at `INFO` level. + + + **Example usage:** + + ```bash + dbt run-operation grant_schema_role_access \ + --args '{"target_schema": "combined", "users_role": "analyst_role"}' + ``` + + arguments: + - name: target_schema + type: string + description: > + The name of the database schema to grant access to. Intended for shared, + non-program-specific schemas such as `combined`, `access`, or `fhir_export`. + + + - name: users_role + type: string + description: > + The name of the database role to receive the grants. This shouldbe + an existing role in the target database. + + - name: generate_placeholder_global_ids_table + description: | + Creates an empty placeholder global_ids table with the expected schema. + Used when a downstream pipeline expects the table to exist before real + data is populated. + + Required args: + - schema_name + Optional args: + - table_name (defaults to global_ids) + + Command line usage: + dbt run-operation generate_placeholder_global_ids_table --args '{"schema_name": "inc_brainpower_raw"}' + + Explicit table name: + dbt run-operation generate_placeholder_global_ids_table --args '{"schema_name": "inc_brainpower_raw", "table_name": "global_ids"}' + + - name: store_profile + description: | + Profiles a relation and stores results in a table. + This is the primary ad hoc entry point and supports append-only history: + only changed profile rows are inserted with a new profiled_at timestamp. + + Required args: + - profile_table + + Optional args: + - profile_database (defaults to target.database) + - profile_schema (defaults to target.schema) + - profile_include_columns (list) + - profile_exclude_columns (list) + - profile_where_clause + - profile_group_by (list) + - adhoc_database (defaults to profile_database) + - adhoc_schema (defaults to profile_schema) + - adhoc_table (defaults to profile_) + + Command line usage: + dbt run-operation store_profile --args '{"profile_schema": "dev_inc_brainpower_src", "profile_table": "inc_brainpower_src_datasets"}' + + - name: get_profile + description: | + Backward-compatible wrapper around store_profile. + Accepts an args dictionary and forwards all supported profile options. + + Required args: + - args (dictionary) + + Command line usage: + dbt run-operation get_profile --args '{"profile_schema": "dev_inc_brainpower_src", "profile_table": "inc_brainpower_src_datasets"}' + + - name: run_get_profile + description: | + Backward-compatible wrapper around store_profile. + Accepts an args dictionary and forwards all supported profile options. + + Required args: + - args (dictionary) + + Command line usage: + dbt run-operation run_get_profile --args '{"profile_schema": "dev_inc_brainpower_src", "profile_table": "inc_brainpower_src_datasets"}' + + - name: combined_stb_relations + description: | + Builds a list of ref() relations for a combined union model. + Used by union-style models to collect study-specific stb tables. + + Quickstart: + macros/pipeline/docs/combined_stb_relations_quickstart.md + + Required args: + - table_name + Optional args: + - studies_var (defaults to combined_studies) + + Used from model SQL, for example: + combined_stb_relations(table_name='patient') + + - name: combined_union_from_current_model + description: | + Unions the study-specific relations that correspond to the current + combined_ model. Intended for use inside combined union model SQL. + + Quickstart: + macros/pipeline/docs/combined_union_from_current_model_quickstart.md + + Optional args: + - studies_var (defaults to combined_studies) + + Used from model SQL, for example: + combined_union_from_current_model() + + - name: generate_descriptor_view_sql + description: | + Generates placeholder SQL for a descriptor view based on program and + resource exceptions. + Used by descriptor-view models that need a compile-safe SQL stub. + + Quickstart: + macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md + + Required args: + - program_id + - dewrangle_study_id + Optional args: + - inclusion_resources (list) + - exclusion_resources (list) + + Used from model SQL, for example: + generate_descriptor_view_sql(program_id='inc', dewrangle_study_id='brainpower', exclusion_resources=[]) + + - name: generate_schema_name + description: | + Custom schema naming macro used by dbt for seeds and models. + It prefixes the target schema and appends the model folder path so dev + runs land in predictable schema names. + + Quickstart: + macros/pipeline/docs/generate_schema_name_quickstart.md + + Used automatically by dbt; not called directly in model SQL. + + - name: generate_stb_sql + description: | + Generates SQL that joins descriptor sources to GID lookup rows. + Used by STB-style models to map descriptors to identifiers. + + Quickstart: + macros/pipeline/docs/generate_stb_sql_quickstart.md + + Required args: + - base_source + Optional args: + - gid_lookup + - descriptor_sources (list) + + Used from model SQL, for example: + generate_stb_sql(base_source='my_base_model', gid_lookup='my_gid_lookup', descriptor_sources=[{'Patient': {} }]) + + - name: normalize_descriptors + description: | + Returns a normalized concatenated SQL expression for one or more + descriptor columns. + Used when building stable composite descriptors from multiple fields. + + Quickstart: + macros/pipeline/docs/normalize_descriptors_quickstart.md + + Default behavior per descriptor entry: + - trims leading and trailing whitespace + - collapses internal whitespace to a single space + - preserves case + - preserves special characters + - escapes the chosen delimiter + - converts blank strings to null + + Keep the common case simple by passing a list of SQL expressions: + normalize_descriptors(descriptor_cols=['id', 'study_code']) + + For overrides, pass a list item as a dictionary with expr plus only the + options you need to change: + normalize_descriptors( + descriptor_cols=[ + 'id', + {'expr': 'file_name', 'escape_delimiter': false}, + {'expr': 'normalized_hpo_term', 'keep_case': false} + ], + delimiter=';' + ) + + A fixed descriptor string can also be normalized with its own override + options: + normalize_descriptors( + descriptor_str='brainpower,v2', + descriptor_str_options={'escape_delimiter': true, 'keep_case': true} + ) + + Required args: + - descriptor_cols (list) + Optional args: + - descriptor_str + - delimiter (defaults to ,) + - default_options (dictionary) + - descriptor_str_options (dictionary) diff --git a/dbt_project/macros/grants/grant_devs_access.sql b/dbt_project/macros/grants/grant_devs_access.sql index 5e6172d..cc80aaa 100644 --- a/dbt_project/macros/grants/grant_devs_access.sql +++ b/dbt_project/macros/grants/grant_devs_access.sql @@ -1,5 +1,13 @@ {% macro grant_devs_access(tag, users_role) %} {% if execute %} + {# + Use explicit table privileges instead of GRANT ALL to follow least-privilege. + In Postgres, ALL TABLE privileges also include TRIGGER, REFERENCES, and TRUNCATE, + which are broader than required for this workflow. + #} + {% set table_privileges = 'select, insert, update, delete' %} + {% set sequence_privileges = 'usage, select' %} + {% set command_name = flags.WHICH if flags is defined and flags.WHICH is defined else none %} {% if command_name not in ['run', 'build'] %} {% do log('grant_devs_access: skipping for command ' ~ (command_name or 'unknown') ~ '; grants are only applied for run/build', info=True) %} @@ -49,11 +57,11 @@ {% do log('Applying ' ~ grantee_name ~ ' grants on schema ' ~ schema_name, info=True) %} {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("grant select, insert, update, delete on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("grant usage, select on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant " ~ table_privileges ~ " on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant " ~ sequence_privileges ~ " on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant select, insert, update, delete on tables to " ~ grantee) %} - {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant usage, select on sequences to " ~ grantee) %} + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ table_privileges ~ " on tables to " ~ grantee) %} + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ sequence_privileges ~ " on sequences to " ~ grantee) %} {% else %} {% do log('Skipping grants for schema ' ~ schema_name ~ ' (already granted)', info=True) %} {% endif %} diff --git a/dbt_project/macros/grants/grant_schema_role_access.sql b/dbt_project/macros/grants/grant_schema_role_access.sql index 03a1cd3..a4bcda1 100644 --- a/dbt_project/macros/grants/grant_schema_role_access.sql +++ b/dbt_project/macros/grants/grant_schema_role_access.sql @@ -1,5 +1,13 @@ {% macro grant_schema_role_access(target_schema, users_role) %} {% if execute %} + {# + Use explicit table privileges instead of GRANT ALL to follow least-privilege. + In Postgres, ALL TABLE privileges also include TRIGGER, REFERENCES, and TRUNCATE, + which are broader than required for this workflow. + #} + {% set table_privileges = 'select, insert, update, delete' %} + {% set sequence_privileges = 'usage, select' %} + {% if not target_schema %} {% do exceptions.raise_compiler_error('grant_schema_role_access: target_schema is required') %} {% endif %} @@ -27,11 +35,11 @@ {% do log('Applying ' ~ grantee_name ~ ' grants on schema ' ~ schema_name, info=True) %} {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("grant select, insert, update, delete on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("grant usage, select on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant " ~ table_privileges ~ " on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant " ~ sequence_privileges ~ " on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} - {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant select, insert, update, delete on tables to " ~ grantee) %} - {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant usage, select on sequences to " ~ grantee) %} + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ table_privileges ~ " on tables to " ~ grantee) %} + {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ sequence_privileges ~ " on sequences to " ~ grantee) %} {% else %} {% do log('Skipping grants for schema ' ~ schema_name ~ ' (already granted for role ' ~ grantee_name ~ ')', info=True) %} {% endif %} diff --git a/dbt_project/macros/pipeline/combined_union.sql b/dbt_project/macros/pipeline/combined_union.sql index b59d754..37341cb 100644 --- a/dbt_project/macros/pipeline/combined_union.sql +++ b/dbt_project/macros/pipeline/combined_union.sql @@ -1,21 +1,21 @@ -{% macro combined_stb_relations(table_name, studies_var='combined_studies') %} - {% set studies = var(studies_var, []) %} +{%- macro combined_stb_relations(table_name, studies_var='combined_studies') -%} + {%- set studies = var(studies_var, []) -%} - {% if studies | length == 0 %} + {%- if studies | length == 0 -%} {{ exceptions.raise_compiler_error("Var '" ~ studies_var ~ "' must contain at least one study prefix.") }} - {% endif %} + {%- endif -%} - {% set relations = [] %} - {% for study in studies %} - {% do relations.append(ref(study ~ '_stb_' ~ table_name)) %} - {% endfor %} + {%- set relations = [] -%} + {%- for study in studies -%} + {%- do relations.append(ref(study ~ '_stb_' ~ table_name)) -%} + {%- endfor -%} - {{ return(relations) }} -{% endmacro %} + {{- return(relations) -}} +{%- endmacro -%} -{% macro combined_union_from_current_model(studies_var='combined_studies') %} - {% set table_name = model.name | replace('combined_', '') %} - {% set relations = combined_stb_relations(table_name=table_name, studies_var=studies_var) %} +{%- macro combined_union_from_current_model(studies_var='combined_studies') -%} + {%- set table_name = model.name | replace('combined_', '') -%} + {%- set relations = combined_stb_relations(table_name=table_name, studies_var=studies_var) -%} - {{ dbt_utils.union_relations(relations=relations) }} -{% endmacro %} + {{- dbt_utils.union_relations(relations=relations) -}} +{%- endmacro -%} From 8e1d5523a0a2ab0c55228445727d69247b46e7b0 Mon Sep 17 00:00:00 2001 From: brendagutman Date: Thu, 23 Jul 2026 14:20:59 -0500 Subject: [PATCH 10/11] Fix the _macros yml --- dbt_project/macros/_macros.yml | 185 --------------------------------- 1 file changed, 185 deletions(-) diff --git a/dbt_project/macros/_macros.yml b/dbt_project/macros/_macros.yml index 21ffcc0..1b136ca 100644 --- a/dbt_project/macros/_macros.yml +++ b/dbt_project/macros/_macros.yml @@ -306,188 +306,3 @@ macros: description: > The name of the database role to receive the grants. This shouldbe an existing role in the target database. - - - name: generate_placeholder_global_ids_table - description: | - Creates an empty placeholder global_ids table with the expected schema. - Used when a downstream pipeline expects the table to exist before real - data is populated. - - Required args: - - schema_name - Optional args: - - table_name (defaults to global_ids) - - Command line usage: - dbt run-operation generate_placeholder_global_ids_table --args '{"schema_name": "inc_brainpower_raw"}' - - Explicit table name: - dbt run-operation generate_placeholder_global_ids_table --args '{"schema_name": "inc_brainpower_raw", "table_name": "global_ids"}' - - - name: store_profile - description: | - Profiles a relation and stores results in a table. - This is the primary ad hoc entry point and supports append-only history: - only changed profile rows are inserted with a new profiled_at timestamp. - - Required args: - - profile_table - - Optional args: - - profile_database (defaults to target.database) - - profile_schema (defaults to target.schema) - - profile_include_columns (list) - - profile_exclude_columns (list) - - profile_where_clause - - profile_group_by (list) - - adhoc_database (defaults to profile_database) - - adhoc_schema (defaults to profile_schema) - - adhoc_table (defaults to profile_) - - Command line usage: - dbt run-operation store_profile --args '{"profile_schema": "dev_inc_brainpower_src", "profile_table": "inc_brainpower_src_datasets"}' - - - name: get_profile - description: | - Backward-compatible wrapper around store_profile. - Accepts an args dictionary and forwards all supported profile options. - - Required args: - - args (dictionary) - - Command line usage: - dbt run-operation get_profile --args '{"profile_schema": "dev_inc_brainpower_src", "profile_table": "inc_brainpower_src_datasets"}' - - - name: run_get_profile - description: | - Backward-compatible wrapper around store_profile. - Accepts an args dictionary and forwards all supported profile options. - - Required args: - - args (dictionary) - - Command line usage: - dbt run-operation run_get_profile --args '{"profile_schema": "dev_inc_brainpower_src", "profile_table": "inc_brainpower_src_datasets"}' - - - name: combined_stb_relations - description: | - Builds a list of ref() relations for a combined union model. - Used by union-style models to collect study-specific stb tables. - - Quickstart: - macros/pipeline/docs/combined_stb_relations_quickstart.md - - Required args: - - table_name - Optional args: - - studies_var (defaults to combined_studies) - - Used from model SQL, for example: - combined_stb_relations(table_name='patient') - - - name: combined_union_from_current_model - description: | - Unions the study-specific relations that correspond to the current - combined_ model. Intended for use inside combined union model SQL. - - Quickstart: - macros/pipeline/docs/combined_union_from_current_model_quickstart.md - - Optional args: - - studies_var (defaults to combined_studies) - - Used from model SQL, for example: - combined_union_from_current_model() - - - name: generate_descriptor_view_sql - description: | - Generates placeholder SQL for a descriptor view based on program and - resource exceptions. - Used by descriptor-view models that need a compile-safe SQL stub. - - Quickstart: - macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md - - Required args: - - program_id - - dewrangle_study_id - Optional args: - - inclusion_resources (list) - - exclusion_resources (list) - - Used from model SQL, for example: - generate_descriptor_view_sql(program_id='inc', dewrangle_study_id='brainpower', exclusion_resources=[]) - - - name: generate_schema_name - description: | - Custom schema naming macro used by dbt for seeds and models. - It prefixes the target schema and appends the model folder path so dev - runs land in predictable schema names. - - Quickstart: - macros/pipeline/docs/generate_schema_name_quickstart.md - - Used automatically by dbt; not called directly in model SQL. - - - name: generate_stb_sql - description: | - Generates SQL that joins descriptor sources to GID lookup rows. - Used by STB-style models to map descriptors to identifiers. - - Quickstart: - macros/pipeline/docs/generate_stb_sql_quickstart.md - - Required args: - - base_source - Optional args: - - gid_lookup - - descriptor_sources (list) - - Used from model SQL, for example: - generate_stb_sql(base_source='my_base_model', gid_lookup='my_gid_lookup', descriptor_sources=[{'Patient': {} }]) - - - name: normalize_descriptors - description: | - Returns a normalized concatenated SQL expression for one or more - descriptor columns. - Used when building stable composite descriptors from multiple fields. - - Quickstart: - macros/pipeline/docs/normalize_descriptors_quickstart.md - - Default behavior per descriptor entry: - - trims leading and trailing whitespace - - collapses internal whitespace to a single space - - preserves case - - preserves special characters - - escapes the chosen delimiter - - converts blank strings to null - - Keep the common case simple by passing a list of SQL expressions: - normalize_descriptors(descriptor_cols=['id', 'study_code']) - - For overrides, pass a list item as a dictionary with expr plus only the - options you need to change: - normalize_descriptors( - descriptor_cols=[ - 'id', - {'expr': 'file_name', 'escape_delimiter': false}, - {'expr': 'normalized_hpo_term', 'keep_case': false} - ], - delimiter=';' - ) - - A fixed descriptor string can also be normalized with its own override - options: - normalize_descriptors( - descriptor_str='brainpower,v2', - descriptor_str_options={'escape_delimiter': true, 'keep_case': true} - ) - - Required args: - - descriptor_cols (list) - Optional args: - - descriptor_str - - delimiter (defaults to ,) - - default_options (dictionary) - - descriptor_str_options (dictionary) From 8b3a59de7b17fdf31bf5e46d6008b81b38e7be12 Mon Sep 17 00:00:00 2001 From: brendagutman Date: Thu, 23 Jul 2026 15:01:22 -0500 Subject: [PATCH 11/11] Ensure the create permissions are set properly --- dbt_project/analyses/grants/_analysis.yml | 28 +++++++++++++++++ .../grants/check_schema_role_privileges.sql | 30 +++++++++++++++++++ dbt_project/macros/_macros.yml | 10 +++++-- .../macros/grants/grant_devs_access.sql | 13 +++++--- .../grants/grant_schema_role_access.sql | 15 ++++++---- 5 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 dbt_project/analyses/grants/_analysis.yml create mode 100644 dbt_project/analyses/grants/check_schema_role_privileges.sql diff --git a/dbt_project/analyses/grants/_analysis.yml b/dbt_project/analyses/grants/_analysis.yml new file mode 100644 index 0000000..e0e9dea --- /dev/null +++ b/dbt_project/analyses/grants/_analysis.yml @@ -0,0 +1,28 @@ +version: 2 + +analyses: + - name: check_schema_role_privileges + description: | + Lists schema-level privileges for roles across non-system schemas. + Useful for validating whether grant macros are correctly applying + CREATE and USAGE permissions. + + Optional vars: + - grants_schema_like (default: %brainpower%) + - grants_role_like (default: %include%) + + Example dbt show: + dbt show --select path:analyses/grants/check_schema_role_privileges.sql + + Example dbt show with vars: + dbt show --select path:analyses/grants/check_schema_role_privileges.sql --vars '{"grants_schema_like":"dev_inc_%","grants_role_like":"include%"}' + + columns: + - name: schema_name + description: Name of the schema being inspected. + - name: role_name + description: Name of the role being inspected. + - name: can_create + description: YES when the role has CREATE on the schema; otherwise NO. + - name: can_usage + description: YES when the role has USAGE on the schema; otherwise NO. diff --git a/dbt_project/analyses/grants/check_schema_role_privileges.sql b/dbt_project/analyses/grants/check_schema_role_privileges.sql new file mode 100644 index 0000000..5fa93d8 --- /dev/null +++ b/dbt_project/analyses/grants/check_schema_role_privileges.sql @@ -0,0 +1,30 @@ +-- PostgreSQL: List roles and their privileges on each schema +-- Optional vars: +-- grants_schema_like (default: %brainpower%) +-- grants_role_like (default: %include%) + +{% set grants_schema_like = var('grants_schema_like', '%brainpower%') %} +{% set grants_role_like = var('grants_role_like', '%include%') %} + +SELECT + n.nspname AS schema_name, + r.rolname AS role_name, + CASE + WHEN has_schema_privilege(r.rolname, n.nspname, 'CREATE') THEN 'YES' + ELSE 'NO' + END AS can_create, + CASE + WHEN has_schema_privilege(r.rolname, n.nspname, 'USAGE') THEN 'YES' + ELSE 'NO' + END AS can_usage +FROM + pg_namespace n +CROSS JOIN + pg_roles r +WHERE + n.nspname NOT LIKE 'pg_%' + AND n.nspname <> 'information_schema' + AND n.nspname LIKE '{{ grants_schema_like }}' + AND r.rolname LIKE '{{ grants_role_like }}' +ORDER BY + schema_name, role_name diff --git a/dbt_project/macros/_macros.yml b/dbt_project/macros/_macros.yml index 1b136ca..e8a7ad4 100644 --- a/dbt_project/macros/_macros.yml +++ b/dbt_project/macros/_macros.yml @@ -208,10 +208,12 @@ macros: that tag), logs an INFO message and exits without issuing any SQL. 4. For each collected schema, checks `has_schema_privilege(..., 'USAGE')`. - Already-granted schemas are skipped with an INFO log. + and `has_schema_privilege(..., 'CREATE')`. + Schemas that already have both privileges are skipped with an INFO log. 5. For schemas that still need granting, issues: - `GRANT USAGE ON SCHEMA` — allows the role to see and interact with the schema. + - `GRANT CREATE ON SCHEMA` — allows the role to create objects in the schema. - `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA` — full DML access on every table that currently exists in the schema. - `GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA` — access to all existing @@ -267,13 +269,15 @@ macros: compiler error if either is missing. 2. Checks whether the role already holds `USAGE` privilege on the schema using - `has_schema_privilege`. If access is already granted, the macro logs a skip - message and exits without issuing any SQL — making it safe to run repeatedly + `has_schema_privilege`, and also checks `CREATE` on the same schema. + If both are already granted, the macro logs a skip + message and exits without issuing grant SQL — making it safe to run repeatedly without accumulating duplicate grants. 3. If access is not yet granted, the macro issues the following statements against the target schema: - `GRANT USAGE ON SCHEMA` — allows the role to see and interact with the schema. + - `GRANT CREATE ON SCHEMA` — allows the role to create objects in the schema. - `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA` — full DML access on every table that currently exists in the schema. - `GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA` — access to all existing diff --git a/dbt_project/macros/grants/grant_devs_access.sql b/dbt_project/macros/grants/grant_devs_access.sql index cc80aaa..f90cdae 100644 --- a/dbt_project/macros/grants/grant_devs_access.sql +++ b/dbt_project/macros/grants/grant_devs_access.sql @@ -44,19 +44,24 @@ {% for schema_name in run_schemas %} {% set quoted_schema = adapter.quote(schema_name) %} {% set schema_granted_sql %} - select has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') + select + has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') as has_usage, + has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'CREATE') as has_create {% endset %} {% set schema_granted_result = run_query(schema_granted_sql) %} - {% set schema_granted = false %} + {% set schema_usage_granted = false %} + {% set schema_create_granted = false %} {% if schema_granted_result is not none and schema_granted_result.rows | length > 0 %} - {% set schema_granted = schema_granted_result.rows[0][0] %} + {% set schema_usage_granted = schema_granted_result.rows[0][0] %} + {% set schema_create_granted = schema_granted_result.rows[0][1] %} {% endif %} - {% if not schema_granted %} + {% if not schema_usage_granted or not schema_create_granted %} {% do log('Applying ' ~ grantee_name ~ ' grants on schema ' ~ schema_name, info=True) %} {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant create on schema " ~ quoted_schema ~ " to " ~ grantee) %} {% do run_query("grant " ~ table_privileges ~ " on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} {% do run_query("grant " ~ sequence_privileges ~ " on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} diff --git a/dbt_project/macros/grants/grant_schema_role_access.sql b/dbt_project/macros/grants/grant_schema_role_access.sql index a4bcda1..3aaf30e 100644 --- a/dbt_project/macros/grants/grant_schema_role_access.sql +++ b/dbt_project/macros/grants/grant_schema_role_access.sql @@ -22,26 +22,31 @@ {% set grantee = adapter.quote(grantee_name) %} {% set schema_granted_sql %} - select has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') + select + has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') as has_usage, + has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'CREATE') as has_create {% endset %} {% set schema_granted_result = run_query(schema_granted_sql) %} - {% set schema_granted = false %} + {% set schema_usage_granted = false %} + {% set schema_create_granted = false %} {% if schema_granted_result is not none and schema_granted_result.rows | length > 0 %} - {% set schema_granted = schema_granted_result.rows[0][0] %} + {% set schema_usage_granted = schema_granted_result.rows[0][0] %} + {% set schema_create_granted = schema_granted_result.rows[0][1] %} {% endif %} - {% if not schema_granted %} + {% if not schema_usage_granted or not schema_create_granted %} {% do log('Applying ' ~ grantee_name ~ ' grants on schema ' ~ schema_name, info=True) %} {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %} + {% do run_query("grant create on schema " ~ quoted_schema ~ " to " ~ grantee) %} {% do run_query("grant " ~ table_privileges ~ " on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %} {% do run_query("grant " ~ sequence_privileges ~ " on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %} {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ table_privileges ~ " on tables to " ~ grantee) %} {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ sequence_privileges ~ " on sequences to " ~ grantee) %} {% else %} - {% do log('Skipping grants for schema ' ~ schema_name ~ ' (already granted for role ' ~ grantee_name ~ ')', info=True) %} + {% do log('Skipping grants for schema ' ~ schema_name ~ ' (USAGE and CREATE already granted for role ' ~ grantee_name ~ ')', info=True) %} {% endif %} {% endif %}