add dummy rule for duckdb installs

config/config.yaml

@@ -28,8 +28,7 @@ scenarios:
         release_type: "gbOpen"
       BEL:
         subtype: 1
-        source: "geoboundaries"
-        release_type: "gbOpen"
+        source: "gadm"
       CHE:
         subtype: "country"
         source: "overture"

workflow/rules/download.smk

@@ -1,6 +1,21 @@
 """Rules to used to download resource files."""
 
 
+rule download_duckdb_extensions:
+    output:
+        path="<resources>/automatic/overture/duckdb_extensions.txt",
+    log:
+        "<logs>/download_duckdb_extensions.log",
+    localrule: True
+    conda:
+        "../envs/shape.yaml"
+    threads: 1
+    message:
+        "Downloading DuckDB extensions."
+    script:
+        "../scripts/download_duckdb_extensions.py"
+
+
 rule download_geoboundaries:
     output:
         path="<resources>/automatic/geoboundaries/download/{country}_{subtype}_{release_type}.parquet",

workflow/rules/harmonise.smk

@@ -17,6 +17,8 @@ rule harmonise_geoboundaries:
 
 
 rule download_harmonised_overture:
+    input:
+        duckdb_extensions=rules.download_duckdb_extensions.output.path,
     output:
         path="<resources>/automatic/overture/harmonise/{country}_{subtype}.parquet",
     log:

workflow/scripts/download_duckdb_extensions.py

@@ -0,0 +1,28 @@
+"""Download DuckDB extensions needed by datasources using it."""
+
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import duckdb
+
+if TYPE_CHECKING:
+    snakemake: Any
+
+
+def main() -> None:
+    """Install DuckDB extensions."""
+    connection = duckdb.connect()
+    installed_extensions = []
+    for extension in ["spatial", "httpfs"]:
+        connection.install_extension(extension)
+        installed_extensions.append(extension)
+
+    output_path = Path(snakemake.output.path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text("\n".join(installed_extensions) + "\n")
+
+
+if __name__ == "__main__":
+    sys.stderr = open(snakemake.log[0], "w")
+    main()

workflow/scripts/download_harmonised_overture.py

@@ -98,7 +98,6 @@ def download_country_overture(country: str, subtype: str, version: str, path: st
     # Setup SQL connection to the remote dataset
     connection = duckdb.connect()
     for extension in ["spatial", "httpfs"]:
-        connection.install_extension(extension)
         connection.load_extension(extension)
     connection.sql("SET s3_region='us-west-2'")