Feature/fail on bad download

.github/workflows/check-version.yml

@@ -1,9 +1,13 @@
 # Check for changes in the upstream template. If changes are found, an issue is created
 name: Template check.
 on:
+  workflow_dispatch:
   schedule:
     - cron: '0 0 1 * *' # Runs at 00:00 UTC on the 1st day of every month
 
 jobs:
   copier-update:
+    permissions:
+      contents: read
+      issues: write
     uses: modelblocks-org/data-module-template/.github/workflows/template-check-version.yml@latest

.github/workflows/pr-ci.yml

@@ -33,7 +33,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: integration-test-logs-${{ matrix.os }}
-          path: tests/integration/resources/powerplants/logs
+          path: tests/integration/resources/module/logs
           if-no-files-found: ignore
           retention-days: 30
       - name: Fail if integration or linting failed

tests/integration/Snakefile

@@ -18,9 +18,9 @@ module module_powerplants:
         aggregated_capacity="results/outputs/{shapes}/aggregated/{adjustment}/{category}.parquet",
         powerplants="results/outputs/{shapes}/powerplants/{adjustment}/{category}.parquet",
         # Redirect module intermediate files
-        logs="resources/powerplants/logs",
-        resources="resources/powerplants/resources",
-        results="resources/powerplants/results",
+        logs="resources/module/logs",
+        resources="resources/module/resources",
+        results="resources/module/results",
     snakefile:
         "../../workflow/Snakefile"
     config:

workflow/internal/settings.yaml

@@ -1,4 +1,6 @@
 # Module settings that users cannot modify.
+curl:
+  retry: 5
 crs:
   geographic: "EPSG:4326"
 resources:

workflow/rules/automatic.smk

@@ -9,12 +9,13 @@ rule download_eia:
     conda:
         "../envs/shell.yaml"
     params:
+        retry=internal["curl"]["retry"],
         url=internal["resources"]["automatic"]["EIA"],
     message:
         "Download the EIA International energy statistics in bulk."
     shell:
         """
-        curl -sSLo {output.path:q} {params.url:q}
+        curl -fsSL --retry {params.retry:q} -o {output.path:q} {params.url:q}
         """
 
 
@@ -26,12 +27,13 @@ rule download_tz_sam:
     conda:
         "../envs/shell.yaml"
     params:
+        retry=internal["curl"]["retry"],
         url=internal["resources"]["automatic"]["TZ-SAM"],
     message:
         "Download the Transition Zero - Solar Asset Mapper dataset."
     shell:
         """
-        curl -sSLo {output.path:q} {params.url:q}
+        curl -fsSL --retry {params.retry:q} -o {output.path:q} {params.url:q}
         """
 
 
@@ -43,12 +45,13 @@ rule download_glohydrores:
     conda:
         "../envs/shell.yaml"
     params:
+        retry=internal["curl"]["retry"],
         url=internal["resources"]["automatic"]["GloHydroRes"],
     message:
         "Download the GloHydroRes dataset."
     shell:
         """
-        curl -sSLo {output.path:q} {params.url:q}
+        curl -fsSL --retry {params.retry:q} -o {output.path:q} {params.url:q}
         """
 
 
@@ -60,10 +63,11 @@ rule download_gem:
     conda:
         "../envs/shell.yaml"
     params:
+        retry=internal["curl"]["retry"],
         url=lambda wc: internal["resources"]["automatic"]["GEM"][wc.dataset],
     message:
         "Download the GEM {wildcards.dataset} dataset."
     shell:
         """
-        curl -sSLo {output.path:q} {params.url:q}
+        curl -fsSL --retry {params.retry:q} -o {output.path:q} {params.url:q}
         """