From ca3ef04fe3e3acea74b0fb8fc9554b7ce43fe247 Mon Sep 17 00:00:00 2001 From: Niko Aarnio Date: Tue, 4 Feb 2025 10:22:48 +0200 Subject: [PATCH 01/21] fix: fix raster dtypes in calculate weights CLI function --- eis_toolkit/cli.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/eis_toolkit/cli.py b/eis_toolkit/cli.py index a723a99c..9ff99b5d 100644 --- a/eis_toolkit/cli.py +++ b/eis_toolkit/cli.py @@ -3136,11 +3136,19 @@ def weights_of_evidence_calculate_weights_cli( out_rasters_dict = {} file_name = input_raster.name.split(".")[0] + raster_meta.pop("dtype") # Remove dtype from metadata to set it individually + for key, array in arrays.items(): + # Set correct dtype for the array + if key in ["Class", "Pixel count", "Deposit count"]: + dtype = np.uint8 + else: + dtype = np.float32 + array = nan_to_nodata(array, raster_meta["nodata"]) output_raster_name = file_name + "_weights_" + weights_type + "_" + key output_raster_path = output_dir.joinpath(output_raster_name + ".tif") - with rasterio.open(output_raster_path, "w", **raster_meta) as dst: + with rasterio.open(output_raster_path, "w", dtype=dtype, **raster_meta) as dst: dst.write(array, 1) out_rasters_dict[output_raster_name] = str(output_raster_path) From 6d2ae006bd73bf87d2f00d58a190c937e96720f7 Mon Sep 17 00:00:00 2001 From: mipeso Date: Tue, 4 Feb 2025 12:04:24 +0200 Subject: [PATCH 02/21] Add check Makes sure nr_of_deposits is not larger than nr_of_pixels --- eis_toolkit/prediction/weights_of_evidence.py | 6 ++++++ tests/prediction/weights_of_evidence_test.py | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index e9b84d00..ce51f6d2 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -498,7 +498,13 @@ def weights_of_evidence_calculate_responses( Array of posterior probabilites. Array of standard deviations in the posterior probability calculations. Array of confidence of the prospectivity values obtained in the posterior probability array. + + Raises: + InvalidParameterValueException: nr_of_deposits is not smaller than nr_of_pixels. """ + + if nr_of_deposits > nr_of_pixels: + raise InvalidParameterValueException(f"nr_of_deposits > nr_of_pixels: {nr_of_deposits} > {nr_of_pixels}") gen_weights_sum = sum( [ item[GENERALIZED_WEIGHT_PLUS_COLUMN] diff --git a/tests/prediction/weights_of_evidence_test.py b/tests/prediction/weights_of_evidence_test.py index 5e3ae592..7affa6d2 100644 --- a/tests/prediction/weights_of_evidence_test.py +++ b/tests/prediction/weights_of_evidence_test.py @@ -6,7 +6,7 @@ import pytest import rasterio -from eis_toolkit.exceptions import InvalidColumnException +from eis_toolkit.exceptions import InvalidColumnException, InvalidParameterValueException from eis_toolkit.prediction.weights_of_evidence import ( CLASS_COLUMN, CONTRAST_COLUMN, @@ -18,6 +18,7 @@ WEIGHT_S_MINUS_COLUMN, WEIGHT_S_PLUS_COLUMN, generalize_weights_cumulative, + weights_of_evidence_calculate_responses, weights_of_evidence_calculate_weights, ) from eis_toolkit.warnings import ClassificationFailedWarning @@ -165,3 +166,14 @@ def test_cumulative_reclassification_max_studentized_contrast(): result = generalize_weights_cumulative(df, "max_studentized_contrast") assert GENERALIZED_CLASS_COLUMN not in result.columns.values + + +def test_calculate_responses_invalid_nr_of_deposits(): + """Tests that an exception is raises if nr_of_deposits is larger than nr_of_pixels.""" + df = weights_table.copy() + output_arrays = [ + {WEIGHT_PLUS_COLUMN: df[WEIGHT_PLUS_COLUMN].to_numpy()}, + {WEIGHT_S_PLUS_COLUMN: df[WEIGHT_S_PLUS_COLUMN].to_numpy()}, + ] + with pytest.raises(InvalidParameterValueException): + weights_of_evidence_calculate_responses(output_arrays, nr_of_deposits=2, nr_of_pixels=1) From 425fb3b033949ee3a01d1ed3cdd66fae143748ab Mon Sep 17 00:00:00 2001 From: mipeso Date: Tue, 4 Feb 2025 12:30:43 +0200 Subject: [PATCH 03/21] Enable providing deposit data as a raster --- eis_toolkit/prediction/weights_of_evidence.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index ce51f6d2..e9e6aeb4 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -6,7 +6,7 @@ import pandas as pd import rasterio from beartype import beartype -from beartype.typing import Dict, List, Literal, Optional, Sequence, Tuple +from beartype.typing import Dict, List, Literal, Optional, Sequence, Tuple, Union from eis_toolkit.exceptions import ClassificationFailedException, InvalidColumnException, InvalidParameterValueException from eis_toolkit.vector_processing.rasterize_vector import rasterize_vector @@ -68,7 +68,7 @@ } -def _read_and_preprocess_evidence( +def _read_and_preprocess_raster_data( raster: rasterio.io.DatasetReader, nodata: Optional[Number] = None, band: int = 1 ) -> np.ndarray: """Read raster data and handle NoData values.""" @@ -349,7 +349,7 @@ def generalize_weights_cumulative( @beartype def weights_of_evidence_calculate_weights( evidential_raster: rasterio.io.DatasetReader, - deposits: gpd.GeoDataFrame, + deposits: Union[gpd.GeoDataFrame, rasterio.io.DatasetReader], raster_nodata: Optional[Number] = None, weights_type: Literal["unique", "categorical", "ascending", "descending"] = "unique", studentized_contrast_threshold: Number = 1, @@ -360,7 +360,7 @@ def weights_of_evidence_calculate_weights( Args: evidential_raster: The evidential raster. - deposits: Vector data representing the mineral deposits or occurences point data. + deposits: Vector or raster data representing the mineral deposits or occurences point data. raster_nodata: If nodata value of raster is wanted to specify manually. Optional parameter, defaults to None (nodata from raster metadata is used). weights_type: Accepted values are 'unique', 'categorical', 'ascending' and 'descending'. @@ -409,13 +409,16 @@ def weights_of_evidence_calculate_weights( metrics_to_arrays = arrays_to_generate.copy() # 1. Preprocess data - evidence_array = _read_and_preprocess_evidence(evidential_raster, raster_nodata) + evidence_array = _read_and_preprocess_raster_data(evidential_raster, raster_nodata) raster_meta = evidential_raster.meta - # Rasterize deposits - deposit_array = rasterize_vector( - geodataframe=deposits, raster_profile=raster_meta, default_value=1.0, fill_value=0.0 - ) + # Rasterize deposits if vector data + if isinstance(deposits, gpd.GeoDataFrame): + deposit_array = rasterize_vector( + geodataframe=deposits, raster_profile=raster_meta, default_value=1.0, fill_value=0.0 + ) + else: + deposit_array = _read_and_preprocess_raster_data(deposits, raster_nodata) # Mask NaN out of the array nodata_mask = np.isnan(evidence_array) From a497d094647d6d98bde11a35d7fff80a7d1e981f Mon Sep 17 00:00:00 2001 From: mipeso Date: Tue, 4 Feb 2025 12:41:56 +0200 Subject: [PATCH 04/21] Add check Makes sure either of nr_of_deposits and nr_of_pixels cannot be non-positive values. --- eis_toolkit/prediction/weights_of_evidence.py | 7 ++++++- tests/prediction/weights_of_evidence_test.py | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index e9e6aeb4..83af0c03 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -503,11 +503,16 @@ def weights_of_evidence_calculate_responses( Array of confidence of the prospectivity values obtained in the posterior probability array. Raises: - InvalidParameterValueException: nr_of_deposits is not smaller than nr_of_pixels. + InvalidParameterValueException: nr_of_deposits is not smaller than nr_of_pixel, + or at least one of nr_of_deposits and nr_of_pixels is not a positive number. """ + if nr_of_deposits <= 0 or nr_of_pixels <= 0: + raise InvalidParameterValueException("nr_of_deposits and nr_of_pixels must be positive.") + if nr_of_deposits > nr_of_pixels: raise InvalidParameterValueException(f"nr_of_deposits > nr_of_pixels: {nr_of_deposits} > {nr_of_pixels}") + gen_weights_sum = sum( [ item[GENERALIZED_WEIGHT_PLUS_COLUMN] diff --git a/tests/prediction/weights_of_evidence_test.py b/tests/prediction/weights_of_evidence_test.py index 7affa6d2..724bf62d 100644 --- a/tests/prediction/weights_of_evidence_test.py +++ b/tests/prediction/weights_of_evidence_test.py @@ -169,7 +169,7 @@ def test_cumulative_reclassification_max_studentized_contrast(): def test_calculate_responses_invalid_nr_of_deposits(): - """Tests that an exception is raises if nr_of_deposits is larger than nr_of_pixels.""" + """Tests that an exception is raised if nr_of_deposits > nr_of_pixels or either value is not positive.""" df = weights_table.copy() output_arrays = [ {WEIGHT_PLUS_COLUMN: df[WEIGHT_PLUS_COLUMN].to_numpy()}, @@ -177,3 +177,9 @@ def test_calculate_responses_invalid_nr_of_deposits(): ] with pytest.raises(InvalidParameterValueException): weights_of_evidence_calculate_responses(output_arrays, nr_of_deposits=2, nr_of_pixels=1) + + with pytest.raises(InvalidParameterValueException): + weights_of_evidence_calculate_responses(output_arrays, nr_of_deposits=1, nr_of_pixels=0) + + with pytest.raises(InvalidParameterValueException): + weights_of_evidence_calculate_responses(output_arrays, nr_of_deposits=-1, nr_of_pixels=1) From ae7983cb61a448e7f8b59f0c8547aa95d846845b Mon Sep 17 00:00:00 2001 From: mipeso Date: Tue, 4 Feb 2025 12:58:14 +0200 Subject: [PATCH 05/21] Enable providing deposit data also as a raster in calculate weights CLI function --- eis_toolkit/cli.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/eis_toolkit/cli.py b/eis_toolkit/cli.py index 9ff99b5d..1b85e3b9 100644 --- a/eis_toolkit/cli.py +++ b/eis_toolkit/cli.py @@ -3091,8 +3091,8 @@ def gamma_overlay_cli(input_rasters: INPUT_FILES_ARGUMENT, output_raster: OUTPUT # WOFE @app.command() def weights_of_evidence_calculate_weights_cli( - input_raster: INPUT_FILE_OPTION, - input_vector: INPUT_FILE_OPTION, + evidential_raster: INPUT_FILE_OPTION, + deposits: INPUT_FILE_OPTION, output_dir: OUTPUT_DIR_OPTION, raster_nodata: Optional[float] = None, weights_type: Annotated[WeightsType, typer.Option(case_sensitive=False)] = WeightsType.unique, @@ -3115,8 +3115,13 @@ def weights_of_evidence_calculate_weights_cli( typer.echo("Progress: 10%") - evidential_raster = rasterio.open(input_raster) - deposits = gpd.read_file(input_vector) + evidential_raster = rasterio.open(evidential_raster) + + if deposits.suffix in (".tif", ".tiff"): + deposits = rasterio.open(deposits) + else: + deposits = gpd.read_file(deposits) + typer.echo("Progress: 25%") if arrays_to_generate == []: @@ -3135,7 +3140,7 @@ def weights_of_evidence_calculate_weights_cli( df.to_csv(output_dir.joinpath("wofe_results.csv")) out_rasters_dict = {} - file_name = input_raster.name.split(".")[0] + file_name = evidential_raster.name.split(".")[0] raster_meta.pop("dtype") # Remove dtype from metadata to set it individually for key, array in arrays.items(): From b75bcc71f3909c4f3bbde6031d4fd637bd2a7948 Mon Sep 17 00:00:00 2001 From: mipeso Date: Tue, 4 Feb 2025 13:56:41 +0200 Subject: [PATCH 06/21] fix(wofe): Fix output raster path creation, add separate parameter for output csv --- eis_toolkit/cli.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/eis_toolkit/cli.py b/eis_toolkit/cli.py index 1b85e3b9..cce588a0 100644 --- a/eis_toolkit/cli.py +++ b/eis_toolkit/cli.py @@ -3093,7 +3093,8 @@ def gamma_overlay_cli(input_rasters: INPUT_FILES_ARGUMENT, output_raster: OUTPUT def weights_of_evidence_calculate_weights_cli( evidential_raster: INPUT_FILE_OPTION, deposits: INPUT_FILE_OPTION, - output_dir: OUTPUT_DIR_OPTION, + output_results_table: OUTPUT_FILE_OPTION, + output_raster_dir: OUTPUT_DIR_OPTION, raster_nodata: Optional[float] = None, weights_type: Annotated[WeightsType, typer.Option(case_sensitive=False)] = WeightsType.unique, studentized_contrast_threshold: float = 1, @@ -3102,6 +3103,9 @@ def weights_of_evidence_calculate_weights_cli( """ Calculate weights of spatial associations. + Save path for resulting CSV is set using --output-results-table parameter. Output rasters are saved to directory + set with --output-raster-dir parameter. + Parameter --studentized-contrast-threshold is used with 'categorical', 'ascending' and 'descending' weight types. Parameter --arrays-to-generate controls which columns in the weights dataframe are returned as arrays. All column @@ -3137,10 +3141,10 @@ def weights_of_evidence_calculate_weights_cli( ) typer.echo("Progress: 75%") - df.to_csv(output_dir.joinpath("wofe_results.csv")) + df.to_csv(output_results_table) out_rasters_dict = {} - file_name = evidential_raster.name.split(".")[0] + file_name = evidential_raster.name.split("/")[-1].split(".")[0] raster_meta.pop("dtype") # Remove dtype from metadata to set it individually for key, array in arrays.items(): @@ -3152,7 +3156,7 @@ def weights_of_evidence_calculate_weights_cli( array = nan_to_nodata(array, raster_meta["nodata"]) output_raster_name = file_name + "_weights_" + weights_type + "_" + key - output_raster_path = output_dir.joinpath(output_raster_name + ".tif") + output_raster_path = output_raster_dir.joinpath(output_raster_name + ".tif") with rasterio.open(output_raster_path, "w", dtype=dtype, **raster_meta) as dst: dst.write(array, 1) out_rasters_dict[output_raster_name] = str(output_raster_path) @@ -3163,7 +3167,8 @@ def weights_of_evidence_calculate_weights_cli( typer.echo(f"Number of deposit pixels: {nr_of_deposits}") typer.echo(f"Number of all evidence pixels: {nr_of_pixels}") typer.echo(f"Output rasters: {json_str}") - typer.echo(f"Weight calculations completed, rasters and CSV saved to {output_dir}.") + typer.echo(f"Weight calculations completed, rasters saved to {output_raster_dir}.") + typer.echo(f"CSV containing results saved to {output_results_table}.") @app.command() From ea1dcb1ba51030cee9b42c656d6a8dd2a860ead1 Mon Sep 17 00:00:00 2001 From: mipeso Date: Wed, 5 Feb 2025 08:11:28 +0200 Subject: [PATCH 07/21] Check raster grids if deposits given as a raster --- eis_toolkit/prediction/weights_of_evidence.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index 83af0c03..e808d3db 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -9,6 +9,8 @@ from beartype.typing import Dict, List, Literal, Optional, Sequence, Tuple, Union from eis_toolkit.exceptions import ClassificationFailedException, InvalidColumnException, InvalidParameterValueException +from eis_toolkit.raster_processing.unifying import unify_raster_grids +from eis_toolkit.utilities.checks.raster import check_raster_grids from eis_toolkit.vector_processing.rasterize_vector import rasterize_vector from eis_toolkit.warnings import ClassificationFailedWarning, InvalidColumnWarning @@ -411,6 +413,7 @@ def weights_of_evidence_calculate_weights( # 1. Preprocess data evidence_array = _read_and_preprocess_raster_data(evidential_raster, raster_nodata) raster_meta = evidential_raster.meta + raster_profile = evidential_raster.profile # Rasterize deposits if vector data if isinstance(deposits, gpd.GeoDataFrame): @@ -418,7 +421,18 @@ def weights_of_evidence_calculate_weights( geodataframe=deposits, raster_profile=raster_meta, default_value=1.0, fill_value=0.0 ) else: - deposit_array = _read_and_preprocess_raster_data(deposits, raster_nodata) + deposit_profile = deposits.profile + + if not check_raster_grids([raster_profile, deposit_profile], same_extent=True): + out_rasters = unify_raster_grids( + base_raster=evidential_raster, + rasters_to_unify=[deposits], + resampling_method="nearest", + masking="extent", + ) + deposit_array = out_rasters[1][0] + else: + deposit_array = _read_and_preprocess_raster_data(deposits, raster_nodata) # Mask NaN out of the array nodata_mask = np.isnan(evidence_array) From 1ef805b8cc880f6b1bca16bccacdf9fd7a9dbc8b Mon Sep 17 00:00:00 2001 From: mipeso Date: Wed, 5 Feb 2025 08:26:32 +0200 Subject: [PATCH 08/21] Fix typo --- eis_toolkit/prediction/weights_of_evidence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index e808d3db..1b7b0204 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -428,7 +428,7 @@ def weights_of_evidence_calculate_weights( base_raster=evidential_raster, rasters_to_unify=[deposits], resampling_method="nearest", - masking="extent", + masking="extents", ) deposit_array = out_rasters[1][0] else: From baa3741135a1c8dbe84a661de93e1ca6e3bdd2e9 Mon Sep 17 00:00:00 2001 From: mipeso Date: Wed, 5 Feb 2025 08:28:00 +0200 Subject: [PATCH 09/21] Fix raster file format check --- eis_toolkit/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eis_toolkit/cli.py b/eis_toolkit/cli.py index cce588a0..266e1865 100644 --- a/eis_toolkit/cli.py +++ b/eis_toolkit/cli.py @@ -3121,7 +3121,7 @@ def weights_of_evidence_calculate_weights_cli( evidential_raster = rasterio.open(evidential_raster) - if deposits.suffix in (".tif", ".tiff"): + if deposits.suffix in (".tif", ".tiff", ".asc", ".img", ".vrt", ".grd"): deposits = rasterio.open(deposits) else: deposits = gpd.read_file(deposits) From 532e7d7280566e244da8b21abc3a135da28eb1c6 Mon Sep 17 00:00:00 2001 From: mipeso Date: Wed, 5 Feb 2025 09:06:03 +0200 Subject: [PATCH 10/21] feat(agterberg-cheng): Add optional parameter for saving summary in text file --- eis_toolkit/cli.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eis_toolkit/cli.py b/eis_toolkit/cli.py index 266e1865..52f05556 100644 --- a/eis_toolkit/cli.py +++ b/eis_toolkit/cli.py @@ -3248,6 +3248,7 @@ def agterberg_cheng_CI_test_cli( input_posterior_probabilities: INPUT_FILE_OPTION, input_posterior_probabilities_std: INPUT_FILE_OPTION, nr_of_deposits: Annotated[int, typer.Option()], + save_summary: Optional[OUTPUT_FILE_OPTION] = None, ): """Perform the conditional independence test presented by Agterberg-Cheng (2002).""" from eis_toolkit.prediction.weights_of_evidence import agterberg_cheng_CI_test @@ -3270,6 +3271,10 @@ def agterberg_cheng_CI_test_cli( nr_of_deposits=nr_of_deposits, ) + if save_summary is not None: + with open(save_summary, "w") as file: + file.write(summary) + typer.echo("Progress: 100%") typer.echo("Conditional independence test completed.") typer.echo(summary) From 55f81ddf415614f49d9196b72baed0b200a88607 Mon Sep 17 00:00:00 2001 From: mipeso Date: Wed, 5 Feb 2025 11:27:41 +0200 Subject: [PATCH 11/21] Read number of deposits and pixels from calculate weights output df --- eis_toolkit/cli.py | 7 ++++--- eis_toolkit/prediction/weights_of_evidence.py | 13 +++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/eis_toolkit/cli.py b/eis_toolkit/cli.py index 52f05556..252d967d 100644 --- a/eis_toolkit/cli.py +++ b/eis_toolkit/cli.py @@ -3175,11 +3175,10 @@ def weights_of_evidence_calculate_weights_cli( def weights_of_evidence_calculate_responses_cli( input_rasters_weights: INPUT_FILES_OPTION, input_rasters_standard_deviations: INPUT_FILES_OPTION, + input_weights_table: INPUT_FILE_OPTION, output_probabilities: OUTPUT_FILE_OPTION, output_probabilities_std: OUTPUT_FILE_OPTION, output_confidence_array: OUTPUT_FILE_OPTION, - nr_of_deposits: Annotated[int, typer.Option()], - nr_of_pixels: Annotated[int, typer.Option()], ): """ Calculate the posterior probabilities for the given generalized weight arrays. @@ -3216,10 +3215,12 @@ def weights_of_evidence_calculate_responses_cli( dict_array.append({"W+": array_W, "S_W+": array_S_W}) + weights_df = pd.read_csv(input_weights_table) + typer.echo("Progress: 25%") posterior_probabilities, posterior_probabilies_std, confidence_array = weights_of_evidence_calculate_responses( - output_arrays=dict_array, nr_of_deposits=nr_of_deposits, nr_of_pixels=nr_of_pixels + output_arrays=dict_array, weights_df=weights_df ) typer.echo("Progress: 75%") diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index 1b7b0204..97336207 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -499,7 +499,7 @@ def weights_of_evidence_calculate_weights( @beartype def weights_of_evidence_calculate_responses( - output_arrays: Sequence[Dict[str, np.ndarray]], nr_of_deposits: int, nr_of_pixels: int + output_arrays: Sequence[Dict[str, np.ndarray]], weights_df: pd.DataFrame ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: """Calculate the posterior probabilities for the given generalized weight arrays. @@ -508,8 +508,8 @@ def weights_of_evidence_calculate_responses( For each dictionary, generalized weight and generalized standard deviation arrays are used and summed together pixel-wise to calculate the posterior probabilities. If generalized arrays are not found, the W+ and S_W+ arrays are used (so if outputs from unique weight calculations are used for this function). - nr_of_deposits: Number of deposit pixels in the input data for weights of evidence calculations. - nr_of_pixels: Number of evidence pixels in the input data for weights of evidence calculations. + weights_df: Output dataframe of WofE calculate weights algorithm. Used for determining number of deposits and + number of pixels. Returns: Array of posterior probabilites. @@ -521,11 +521,8 @@ def weights_of_evidence_calculate_responses( or at least one of nr_of_deposits and nr_of_pixels is not a positive number. """ - if nr_of_deposits <= 0 or nr_of_pixels <= 0: - raise InvalidParameterValueException("nr_of_deposits and nr_of_pixels must be positive.") - - if nr_of_deposits > nr_of_pixels: - raise InvalidParameterValueException(f"nr_of_deposits > nr_of_pixels: {nr_of_deposits} > {nr_of_pixels}") + nr_of_deposits = weights_df["Deposit count"].sum() + nr_of_pixels = weights_df["Pixel count"].sum() gen_weights_sum = sum( [ From f52d4eaf4f9931bad38518336c568be6538afd6b Mon Sep 17 00:00:00 2001 From: mipeso Date: Wed, 5 Feb 2025 11:34:18 +0200 Subject: [PATCH 12/21] Remove old test --- tests/prediction/weights_of_evidence_test.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/tests/prediction/weights_of_evidence_test.py b/tests/prediction/weights_of_evidence_test.py index 724bf62d..5e3ae592 100644 --- a/tests/prediction/weights_of_evidence_test.py +++ b/tests/prediction/weights_of_evidence_test.py @@ -6,7 +6,7 @@ import pytest import rasterio -from eis_toolkit.exceptions import InvalidColumnException, InvalidParameterValueException +from eis_toolkit.exceptions import InvalidColumnException from eis_toolkit.prediction.weights_of_evidence import ( CLASS_COLUMN, CONTRAST_COLUMN, @@ -18,7 +18,6 @@ WEIGHT_S_MINUS_COLUMN, WEIGHT_S_PLUS_COLUMN, generalize_weights_cumulative, - weights_of_evidence_calculate_responses, weights_of_evidence_calculate_weights, ) from eis_toolkit.warnings import ClassificationFailedWarning @@ -166,20 +165,3 @@ def test_cumulative_reclassification_max_studentized_contrast(): result = generalize_weights_cumulative(df, "max_studentized_contrast") assert GENERALIZED_CLASS_COLUMN not in result.columns.values - - -def test_calculate_responses_invalid_nr_of_deposits(): - """Tests that an exception is raised if nr_of_deposits > nr_of_pixels or either value is not positive.""" - df = weights_table.copy() - output_arrays = [ - {WEIGHT_PLUS_COLUMN: df[WEIGHT_PLUS_COLUMN].to_numpy()}, - {WEIGHT_S_PLUS_COLUMN: df[WEIGHT_S_PLUS_COLUMN].to_numpy()}, - ] - with pytest.raises(InvalidParameterValueException): - weights_of_evidence_calculate_responses(output_arrays, nr_of_deposits=2, nr_of_pixels=1) - - with pytest.raises(InvalidParameterValueException): - weights_of_evidence_calculate_responses(output_arrays, nr_of_deposits=1, nr_of_pixels=0) - - with pytest.raises(InvalidParameterValueException): - weights_of_evidence_calculate_responses(output_arrays, nr_of_deposits=-1, nr_of_pixels=1) From 40b1cee3d5a0443f734aea092b99a1e2e61bfb91 Mon Sep 17 00:00:00 2001 From: mipeso Date: Wed, 5 Feb 2025 13:36:46 +0200 Subject: [PATCH 13/21] Make non-matching raster grids raise an error --- eis_toolkit/prediction/weights_of_evidence.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index 97336207..17b74a10 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -8,8 +8,12 @@ from beartype import beartype from beartype.typing import Dict, List, Literal, Optional, Sequence, Tuple, Union -from eis_toolkit.exceptions import ClassificationFailedException, InvalidColumnException, InvalidParameterValueException -from eis_toolkit.raster_processing.unifying import unify_raster_grids +from eis_toolkit.exceptions import ( + ClassificationFailedException, + InvalidColumnException, + InvalidParameterValueException, + NonMatchingRasterMetadataException, +) from eis_toolkit.utilities.checks.raster import check_raster_grids from eis_toolkit.vector_processing.rasterize_vector import rasterize_vector from eis_toolkit.warnings import ClassificationFailedWarning, InvalidColumnWarning @@ -423,16 +427,10 @@ def weights_of_evidence_calculate_weights( else: deposit_profile = deposits.profile - if not check_raster_grids([raster_profile, deposit_profile], same_extent=True): - out_rasters = unify_raster_grids( - base_raster=evidential_raster, - rasters_to_unify=[deposits], - resampling_method="nearest", - masking="extents", - ) - deposit_array = out_rasters[1][0] - else: + if check_raster_grids([raster_profile, deposit_profile], same_extent=True): deposit_array = _read_and_preprocess_raster_data(deposits, raster_nodata) + else: + raise NonMatchingRasterMetadataException("Input rasters should have the same grid properties.") # Mask NaN out of the array nodata_mask = np.isnan(evidence_array) From 374337716c496e0535f473b93dfc2d37148f3750 Mon Sep 17 00:00:00 2001 From: Niko Aarnio Date: Mon, 10 Feb 2025 11:03:17 +0200 Subject: [PATCH 14/21] fix CRS checking --- eis_toolkit/utilities/checks/raster.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/eis_toolkit/utilities/checks/raster.py b/eis_toolkit/utilities/checks/raster.py index a93a32fc..c638e279 100644 --- a/eis_toolkit/utilities/checks/raster.py +++ b/eis_toolkit/utilities/checks/raster.py @@ -3,6 +3,7 @@ import rasterio.transform from beartype import beartype from beartype.typing import Iterable, Sequence, Union +from rasterio.crs import CRS from eis_toolkit.exceptions import InvalidParameterValueException @@ -37,21 +38,25 @@ def check_matching_crs(objects: Iterable) -> bool: Returns: True if everything matches, False if not. """ - epsg_list = [] + crs_list = [] for object in objects: if not isinstance(object, (rasterio.profiles.Profile, dict)): if not object.crs: return False epsg = object.crs.to_epsg() - epsg_list.append(epsg) + crs_list.append(epsg) else: if "crs" in object: - epsg_list.append(object["crs"]) + crs_object = object["crs"] + if type(crs_object) == CRS: + crs_list.append(crs_object.to_epsg()) + else: + crs_list.append(crs_object) else: return False - if len(set(epsg_list)) != 1: + if len(set(crs_list)) != 1: return False return True @@ -119,11 +124,16 @@ def check_raster_grids( Returns: True if gridding and optionally bounds matches, False if not. """ + import typer + if not check_matching_crs(raster_profiles): + typer.echo("Non matching CRS") return False if not check_matching_pixel_alignment(raster_profiles): + typer.echo("Non matching pixel alignment") return False if same_extent and not check_matching_bounds(raster_profiles): + typer.echo("Non matching extent") return False return True From fb82ce5d7363924a5fc2454b5a2076ba3f992431 Mon Sep 17 00:00:00 2001 From: mipeso Date: Mon, 10 Feb 2025 11:38:36 +0200 Subject: [PATCH 15/21] fix(calculate responses): fix reading in data Read number of deposits and pixels as max values (instead of sums) of corresponding columns in the input df. --- eis_toolkit/prediction/weights_of_evidence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index 17b74a10..0958e4c9 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -519,8 +519,8 @@ def weights_of_evidence_calculate_responses( or at least one of nr_of_deposits and nr_of_pixels is not a positive number. """ - nr_of_deposits = weights_df["Deposit count"].sum() - nr_of_pixels = weights_df["Pixel count"].sum() + nr_of_deposits = weights_df["Deposit count"].max() + nr_of_pixels = weights_df["Pixel count"].max() gen_weights_sum = sum( [ From 390f1629bbfd13e98eb59070f4d976ff33a2edf7 Mon Sep 17 00:00:00 2001 From: mipeso Date: Mon, 10 Feb 2025 11:45:12 +0200 Subject: [PATCH 16/21] fix(agterberg-cheng): fix input parameter Instead of user providing number of deposits as int type parameter, read it from input df. --- eis_toolkit/prediction/weights_of_evidence.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index 0958e4c9..ba90a065 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -554,7 +554,7 @@ def weights_of_evidence_calculate_responses( @beartype def agterberg_cheng_CI_test( - posterior_probabilities: np.ndarray, posterior_probabilities_std: np.ndarray, nr_of_deposits: int + posterior_probabilities: np.ndarray, posterior_probabilities_std: np.ndarray, weights_df: pd.DataFrame ) -> Tuple[bool, bool, bool, float, str]: """Perform the conditional independence test presented by Agterberg-Cheng (2002). @@ -564,7 +564,8 @@ def agterberg_cheng_CI_test( Args: posterior_probabilities: Array of posterior probabilites. posterior_probabilities_std: Array of standard deviations in the posterior probability calculations. - nr_of_deposits: Number of deposit pixels in the input data for weights of evidence calculations. + weights_df: Output dataframe of WofE calculate weights algorithm. Used for determining number of deposits. + Returns: Whether the conditional hypothesis can be accepted for the evidence layers that the input posterior probabilities and standard deviations of posterior probabilities are calculated from. @@ -573,12 +574,9 @@ def agterberg_cheng_CI_test( Ratio T/n. Results > 1, may be because of lack of conditional independence of layers. T should not exceed n by more than 15% (Bonham-Carter 1994, p. 316). A summary of the the conditional independence calculations. - - Raises: - InvalidParameterValueException: Value of nr_of_deposits is not at least 1. """ - if nr_of_deposits < 1: - raise InvalidParameterValueException("Expected input deposits count to be at least 1.") + + nr_of_deposits = weights_df["Deposit count"].max() # One-tailed significance test according to Agterberg-Cheng (2002): # Conditional independence must satisfy: From a9b8228d97082aa935793a4b058f5f99d36c77aa Mon Sep 17 00:00:00 2001 From: mipeso Date: Mon, 10 Feb 2025 13:35:52 +0200 Subject: [PATCH 17/21] Remove printing --- eis_toolkit/utilities/checks/raster.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eis_toolkit/utilities/checks/raster.py b/eis_toolkit/utilities/checks/raster.py index c638e279..78b52a66 100644 --- a/eis_toolkit/utilities/checks/raster.py +++ b/eis_toolkit/utilities/checks/raster.py @@ -124,16 +124,12 @@ def check_raster_grids( Returns: True if gridding and optionally bounds matches, False if not. """ - import typer if not check_matching_crs(raster_profiles): - typer.echo("Non matching CRS") return False if not check_matching_pixel_alignment(raster_profiles): - typer.echo("Non matching pixel alignment") return False if same_extent and not check_matching_bounds(raster_profiles): - typer.echo("Non matching extent") return False return True From df4966296317ce39a90eed51df84eefdb0630d6c Mon Sep 17 00:00:00 2001 From: mipeso Date: Mon, 10 Feb 2025 14:10:22 +0200 Subject: [PATCH 18/21] fix(wofe): fix calculating nr_of_deposits and nr_of_pixels Add function to calculate nr of deposits and pixels from weights dataframe using input array. --- eis_toolkit/cli.py | 6 +++-- eis_toolkit/prediction/weights_of_evidence.py | 24 +++++++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/eis_toolkit/cli.py b/eis_toolkit/cli.py index 252d967d..7b5a042a 100644 --- a/eis_toolkit/cli.py +++ b/eis_toolkit/cli.py @@ -3248,7 +3248,7 @@ def weights_of_evidence_calculate_responses_cli( def agterberg_cheng_CI_test_cli( input_posterior_probabilities: INPUT_FILE_OPTION, input_posterior_probabilities_std: INPUT_FILE_OPTION, - nr_of_deposits: Annotated[int, typer.Option()], + input_weights_table: INPUT_FILE_OPTION, save_summary: Optional[OUTPUT_FILE_OPTION] = None, ): """Perform the conditional independence test presented by Agterberg-Cheng (2002).""" @@ -3264,12 +3264,14 @@ def agterberg_cheng_CI_test_cli( posterior_probabilities_std = src.read(1) posterior_probabilities_std = nodata_to_nan(posterior_probabilities_std, src.nodata) + weights_df = pd.read_csv(input_weights_table) + typer.echo("Progress: 25%") _, _, _, _, summary = agterberg_cheng_CI_test( posterior_probabilities=posterior_probabilities, posterior_probabilities_std=posterior_probabilities_std, - nr_of_deposits=nr_of_deposits, + weights_df=weights_df, ) if save_summary is not None: diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index ba90a065..6915e42d 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -250,6 +250,22 @@ def _generate_arrays_from_metrics( return array_dict +def _calculate_nr_of_deposit_pixels(array: np.ndarray, df: pd.DataFrame) -> Tuple[int, int]: + masked_array = array[~np.isnan(array)] + nr_of_pixels = int(np.size(masked_array)) + + pixels_column = df["Pixel count"] + + match = pixels_column == nr_of_pixels + if match.any(): + nr_of_deposits = df.loc[match, "Deposit count"].iloc[0] + else: + nr_of_pixels = df["Pixel count"].sum() + nr_of_deposits = df["Deposit count"].sum() + + return nr_of_deposits, nr_of_pixels + + @beartype def generalize_weights_cumulative( df: pd.DataFrame, @@ -518,9 +534,8 @@ def weights_of_evidence_calculate_responses( InvalidParameterValueException: nr_of_deposits is not smaller than nr_of_pixel, or at least one of nr_of_deposits and nr_of_pixels is not a positive number. """ - - nr_of_deposits = weights_df["Deposit count"].max() - nr_of_pixels = weights_df["Pixel count"].max() + array = list(output_arrays[0].values())[0] + nr_of_deposits, nr_of_pixels = _calculate_nr_of_deposit_pixels(array, weights_df) gen_weights_sum = sum( [ @@ -575,8 +590,7 @@ def agterberg_cheng_CI_test( T should not exceed n by more than 15% (Bonham-Carter 1994, p. 316). A summary of the the conditional independence calculations. """ - - nr_of_deposits = weights_df["Deposit count"].max() + nr_of_deposits, _ = _calculate_nr_of_deposit_pixels(posterior_probabilities, weights_df) # One-tailed significance test according to Agterberg-Cheng (2002): # Conditional independence must satisfy: From e1a8a41a55a133c63f292e4e6225ef666726f25c Mon Sep 17 00:00:00 2001 From: mipeso Date: Mon, 10 Feb 2025 14:14:34 +0200 Subject: [PATCH 19/21] Fix docstring --- eis_toolkit/prediction/weights_of_evidence.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eis_toolkit/prediction/weights_of_evidence.py b/eis_toolkit/prediction/weights_of_evidence.py index 6915e42d..5a21a395 100644 --- a/eis_toolkit/prediction/weights_of_evidence.py +++ b/eis_toolkit/prediction/weights_of_evidence.py @@ -529,10 +529,6 @@ def weights_of_evidence_calculate_responses( Array of posterior probabilites. Array of standard deviations in the posterior probability calculations. Array of confidence of the prospectivity values obtained in the posterior probability array. - - Raises: - InvalidParameterValueException: nr_of_deposits is not smaller than nr_of_pixel, - or at least one of nr_of_deposits and nr_of_pixels is not a positive number. """ array = list(output_arrays[0].values())[0] nr_of_deposits, nr_of_pixels = _calculate_nr_of_deposit_pixels(array, weights_df) From ddf1f706fd2742e085ed86d8f0ac014a6b3a18e6 Mon Sep 17 00:00:00 2001 From: mipeso Date: Tue, 11 Feb 2025 08:22:15 +0200 Subject: [PATCH 20/21] fix: remove prints --- eis_toolkit/cli.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eis_toolkit/cli.py b/eis_toolkit/cli.py index 7b5a042a..ec7f2d6d 100644 --- a/eis_toolkit/cli.py +++ b/eis_toolkit/cli.py @@ -3164,11 +3164,9 @@ def weights_of_evidence_calculate_weights_cli( json_str = json.dumps(out_rasters_dict) typer.echo("Progress 100%") - typer.echo(f"Number of deposit pixels: {nr_of_deposits}") - typer.echo(f"Number of all evidence pixels: {nr_of_pixels}") typer.echo(f"Output rasters: {json_str}") typer.echo(f"Weight calculations completed, rasters saved to {output_raster_dir}.") - typer.echo(f"CSV containing results saved to {output_results_table}.") + typer.echo(f"Weights table saved to {output_results_table}.") @app.command() @@ -3239,7 +3237,7 @@ def weights_of_evidence_calculate_responses_cli( typer.echo("Progress: 100%") typer.echo( - f"Responses calculations finished, writing output rasters to {output_probabilities}, \ + f"Posterior probability calculations finished, output rasters saved to {output_probabilities}, \ {output_probabilities_std} and {output_confidence_array}" ) From f9b81574f68d6ff4b5b3fedd157d29a58cc2a02b Mon Sep 17 00:00:00 2001 From: mipeso Date: Tue, 11 Feb 2025 08:52:23 +0200 Subject: [PATCH 21/21] fix(agterberg-cheng): remove saving results summary into file --- eis_toolkit/cli.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/eis_toolkit/cli.py b/eis_toolkit/cli.py index ec7f2d6d..1ad52c24 100644 --- a/eis_toolkit/cli.py +++ b/eis_toolkit/cli.py @@ -3247,7 +3247,6 @@ def agterberg_cheng_CI_test_cli( input_posterior_probabilities: INPUT_FILE_OPTION, input_posterior_probabilities_std: INPUT_FILE_OPTION, input_weights_table: INPUT_FILE_OPTION, - save_summary: Optional[OUTPUT_FILE_OPTION] = None, ): """Perform the conditional independence test presented by Agterberg-Cheng (2002).""" from eis_toolkit.prediction.weights_of_evidence import agterberg_cheng_CI_test @@ -3272,10 +3271,6 @@ def agterberg_cheng_CI_test_cli( weights_df=weights_df, ) - if save_summary is not None: - with open(save_summary, "w") as file: - file.write(summary) - typer.echo("Progress: 100%") typer.echo("Conditional independence test completed.") typer.echo(summary)