diff --git a/.github/workflows/check-version.yml b/.github/workflows/check-version.yml index 0a4a602..7bde778 100644 --- a/.github/workflows/check-version.yml +++ b/.github/workflows/check-version.yml @@ -1,9 +1,13 @@ # Check for changes in the upstream template. If changes are found, an issue is created name: Template check. on: + workflow_dispatch: schedule: - cron: '0 0 1 * *' # Runs at 00:00 UTC on the 1st day of every month jobs: copier-update: + permissions: + contents: read + issues: write uses: modelblocks-org/data-module-template/.github/workflows/template-check-version.yml@latest diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index b836c1c..0e17ea4 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -33,7 +33,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: integration-test-logs-${{ matrix.os }} - path: tests/integration/resources/powerplants/logs + path: tests/integration/resources/module/logs if-no-files-found: ignore retention-days: 30 - name: Fail if integration or linting failed diff --git a/tests/integration/Snakefile b/tests/integration/Snakefile index 99c95b2..52228d3 100644 --- a/tests/integration/Snakefile +++ b/tests/integration/Snakefile @@ -18,9 +18,9 @@ module module_powerplants: aggregated_capacity="results/outputs/{shapes}/aggregated/{adjustment}/{category}.parquet", powerplants="results/outputs/{shapes}/powerplants/{adjustment}/{category}.parquet", # Redirect module intermediate files - logs="resources/powerplants/logs", - resources="resources/powerplants/resources", - results="resources/powerplants/results", + logs="resources/module/logs", + resources="resources/module/resources", + results="resources/module/results", snakefile: "../../workflow/Snakefile" config: diff --git a/workflow/internal/settings.yaml b/workflow/internal/settings.yaml index 168c0d0..c157a5e 100644 --- a/workflow/internal/settings.yaml +++ b/workflow/internal/settings.yaml @@ -1,4 +1,6 @@ # Module settings that users cannot modify. +curl: + retry: 5 crs: geographic: "EPSG:4326" resources: diff --git a/workflow/rules/automatic.smk b/workflow/rules/automatic.smk index 4c26ac5..4cf846e 100644 --- a/workflow/rules/automatic.smk +++ b/workflow/rules/automatic.smk @@ -9,12 +9,13 @@ rule download_eia: conda: "../envs/shell.yaml" params: + retry=internal["curl"]["retry"], url=internal["resources"]["automatic"]["EIA"], message: "Download the EIA International energy statistics in bulk." shell: """ - curl -sSLo {output.path:q} {params.url:q} + curl -fsSL --retry {params.retry:q} -o {output.path:q} {params.url:q} """ @@ -26,12 +27,13 @@ rule download_tz_sam: conda: "../envs/shell.yaml" params: + retry=internal["curl"]["retry"], url=internal["resources"]["automatic"]["TZ-SAM"], message: "Download the Transition Zero - Solar Asset Mapper dataset." shell: """ - curl -sSLo {output.path:q} {params.url:q} + curl -fsSL --retry {params.retry:q} -o {output.path:q} {params.url:q} """ @@ -43,12 +45,13 @@ rule download_glohydrores: conda: "../envs/shell.yaml" params: + retry=internal["curl"]["retry"], url=internal["resources"]["automatic"]["GloHydroRes"], message: "Download the GloHydroRes dataset." shell: """ - curl -sSLo {output.path:q} {params.url:q} + curl -fsSL --retry {params.retry:q} -o {output.path:q} {params.url:q} """ @@ -60,10 +63,11 @@ rule download_gem: conda: "../envs/shell.yaml" params: + retry=internal["curl"]["retry"], url=lambda wc: internal["resources"]["automatic"]["GEM"][wc.dataset], message: "Download the GEM {wildcards.dataset} dataset." shell: """ - curl -sSLo {output.path:q} {params.url:q} + curl -fsSL --retry {params.retry:q} -o {output.path:q} {params.url:q} """