diff --git a/docs/source/tutorial_load_save.rst b/docs/source/tutorial_load_save.rst index 437f981..a8d9c68 100644 --- a/docs/source/tutorial_load_save.rst +++ b/docs/source/tutorial_load_save.rst @@ -1,6 +1,6 @@ .. meta:: :description lang=en: Tutorial on loading and saving data with ANTS - :keywords: load, save, netcdf, ancil, tutorial + :keywords: load, save, netcdf, ancil, metadata, license, licensing, attribution, references, tutorial :property=og:locale: en_GB .. include:: common.txt @@ -29,10 +29,13 @@ some ANTS specific ones, see :mod:`ants.fileformats`), then additionally: none, and 3. removes any ``forecast_reference_time`` and ``forecast_period`` coordinates, which are not usually wanted in ancillary fields. +4. preserves metadata by loading sidecar metdata files alongside the data + files. :mod:`ants.io.save` similarly extends :py:func:`iris.save` with support for the F03 UM ancillary fileformat and a UKCA specific flavour of NetCDF, neither -of which Iris can write natively. +of which Iris can write natively, and has the option to write metadata files +alongside the saved files in the case of saving an F03 UM ancillary fileformat. Loading data ------------ @@ -82,6 +85,45 @@ a land binary mask field or a land fraction field: "/path/to/target_lsm.nc", land_threshold=0.5 ) +Loading metadata +---------------- +Metadata in this tutorial refers to information about the data in a file, that does not +directly affect that data. For example, licensing information or attributions. + +Metadata in this tutorial refers to information about the data in a file, that does not +directly affect that data. For example, licensing information or attributions. + +Some ancillary file formats cannot include metadata. Metadata can be very important to +keep alongside the data as often data will have some form of requirement or restriction +on it. Because of this, ANTS can handle in external metadata files, provided they match +the naming convention of `filename.extension.accepted-metadata`. The current accepted +metadata attributes are "license", "attribution", "restrictions", "institution", +"acknowledgement", and "references". + +This functionality can be turned off through the use of the argument +``--ignore-metadata-read`` on the command line or when calling the ANTS load: + +.. code-block:: python + + cube = ants.io.load.load("/path/to/source_file" ,ignore_metadata_read=True) + + +.. note:: + Any attributes or files referencing licensing should use the 'license' spelling for + consistency. + +ANTS can load in external metadata files or 'sidecar' files, if they are kept in the +same directory as the data files, and have the same name (including the extension). +Metadata can be loaded alongside all files, including NetCDF files, however ANTS will +not allow you to load a file with a metadata attribute and a sidecar file with the same +attribute. This is to prevent unintentional loss or overwriting of metadata information. + +.. note:: + When using wildcards for loading with sidecar files, add the file extension to the + end to prevent issues with ANTS attempting to load the sidecar files. E.g. + `data*.pp` rather than `data*`. + + Saving data ----------- @@ -114,6 +156,24 @@ data type coercion and attribute renaming) on top of a standard NetCDF save: save.ukca_netcdf(source_cubes, "/path/to/output_ukca.nc") +Saving metadata +--------------- +Metadata will only be saved to a sidecar file, if the ancil saver is used. This is +because NetCDF files will include the attributes within the file. + +This functionality can be turned off through the use of the argument +``--ignore-metadata-write`` on the command line or when calling the ANTS save: + +.. code-block:: python + + import ants.io.save as save + + save.ancil(source_cubes, "/path/to/output_ancil", ignore_metadata_write=True) + +The sidecar files produced by ANTS follow the same naming coventions at those loaded in +`filename.attribute.accepted-metadata`. Only accepted metadata will be written out. + + Key Points ---------- diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 60deaa1..b8c37d5 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -37,16 +37,23 @@ import iris -def load_data(source): +def load_data(source, ignore_metadata_read): with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "NetCDF default loading", iris._deprecation.IrisDeprecation ) - cubes = ants.io.load.load(source) + cubes = ants.io.load.load(source, ignore_metadata_read=ignore_metadata_read) return cubes -def main(source_path, output_path, grid_staggering, netcdf_only): +def main( + source_path, + output_path, + grid_staggering, + netcdf_only, + ignore_metadata_read, + ignore_metadata_write, +): """ Convert specified source file to an ancillary. @@ -58,6 +65,12 @@ def main(source_path, output_path, grid_staggering, netcdf_only): Output ancillary file path. A NetCDF will also be written. grid_staggering : :obj:`int`, optional Grid staggering lookup, see UM F03 - FLH(9). + netcdf_only : bool + If True, only write to netCDF. If False, write to both netCDF and ancillary. + ignore_metadata_read : :obj:`bool`, optional + If True, do not read metadata sidecar files alongside the source. + ignore_metadata_write : :obj:`bool`, optional + If True, do not write metadata sidecar files alongside the ancillary. Returns ------- @@ -66,13 +79,17 @@ def main(source_path, output_path, grid_staggering, netcdf_only): written to an ancillary. """ - source_cubes = load_data(source_path) + source_cubes = load_data(source_path, ignore_metadata_read) if grid_staggering is not None: for source_cube in source_cubes: source_cube.attributes["grid_staggering"] = grid_staggering if not netcdf_only: - save.ancil(source_cubes, output_path) + save.ancil( + source_cubes, + output_path, + ignore_metadata_write=ignore_metadata_write, + ) save.netcdf(source_cubes, output_path) return source_cubes @@ -98,6 +115,8 @@ def cli_interface(): args.output, args.grid_staggering, args.netcdf_only, + args.ignore_metadata_read, + args.ignore_metadata_write, ) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index cf83a4d..adafc96 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -25,6 +25,7 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_read=False, ): """ Load the necessary data for performing a merge and fill operation. @@ -45,6 +46,9 @@ def load_data( Datetime to start the processing. end: :obj:`datetime`, optional Datetime to end the processing. + ignore_metadata_read : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. Returns @@ -56,12 +60,16 @@ def load_data( respectively. """ - primary_cubes = ants.io.load.load(primary_source) + primary_cubes = ants.io.load.load( + primary_source, ignore_metadata_read=ignore_metadata_read + ) if begin is not None: primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end) alternate_cubes = None if alternate_source: - alternate_cubes = ants.io.load.load(alternate_source) + alternate_cubes = ants.io.load.load( + alternate_source, ignore_metadata_read=ignore_metadata_read + ) if begin is not None: alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end) @@ -97,6 +105,8 @@ def main( netcdf_only, search_method, blending_distance, + ignore_metadata_read, + ignore_metadata_write, ): """ Perform merge and fill operation on the provided sources. @@ -152,6 +162,12 @@ def main( is applied. Note that this is in units of grid cells, not a physical distance. If ``None``, no blending is applied, and there will be a hard edge between the two sources. + ignore_metadata_read : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. + ignore_metadata_write : bool + If True, do not write metadata sidecar files alongside the ancillary. + Returns ------- @@ -171,6 +187,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_read, ) result = primary_cubes @@ -182,7 +199,7 @@ def main( ants.analysis.make_consistent_with_lsm(result, lbm, invert_mask, search_method) if not netcdf_only: - save.ancil(result, output) + save.ancil(result, output, ignore_metadata_write=ignore_metadata_write) save.netcdf(result, output) return result @@ -271,6 +288,8 @@ def cli_interface(): netcdf_only=args.netcdf_only, search_method=args.search_method, blending_distance=args.blending_distance, + ignore_metadata_read=args.ignore_metadata_read, + ignore_metadata_write=args.ignore_metadata_write, ) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 7f6bd86..c577e15 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -42,15 +42,20 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_read=False, ): - source_cubes = ants.io.load.load(source) + source_cubes = ants.io.load.load(source, ignore_metadata_read=ignore_metadata_read) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: - target_cube = ants.io.load.load_grid(target_grid) + target_cube = ants.io.load.load_grid( + target_grid, ignore_metadata_read=ignore_metadata_read + ) else: target_cube = ants.io.load.load_landsea_mask( - target_landseamask, land_fraction_threshold + target_landseamask, + land_fraction_threshold, + ignore_metadata_read=ignore_metadata_read, ) return source_cubes, target_cube @@ -76,6 +81,8 @@ def main( save_ukca, netcdf_only, search_method, + ignore_metadata_read, + ignore_metadata_write, ): """ General regrid application top level call function. @@ -118,6 +125,11 @@ def main( provided source(s) consistent with the provided land sea mask. This should only be provided if a target land sea mask is also provided via target_lsm_path. + ignore_metadata_read : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. + ignore_metadata_write : :obj:`bool`, optional + If True, do not write metadata sidecar files alongside the ancillary. Returns ------- @@ -132,6 +144,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_read, ) if ants.utils.cube._is_ugrid(target_cube): raise ValueError( @@ -149,7 +162,11 @@ def main( save.ukca_netcdf(regridded_cubes, output_path) else: if not netcdf_only: - save.ancil(regridded_cubes, output_path) + save.ancil( + regridded_cubes, + output_path, + ignore_metadata_write=ignore_metadata_write, + ) save.netcdf(regridded_cubes, output_path) return regridded_cubes @@ -210,6 +227,8 @@ def cli_interface(): args.save_ukca, args.netcdf_only, args.search_method, + args.ignore_metadata_read, + args.ignore_metadata_write, ) diff --git a/lib/ants/cli/ancil_vertical_regrid.py b/lib/ants/cli/ancil_vertical_regrid.py index c258cbe..b412ae2 100755 --- a/lib/ants/cli/ancil_vertical_regrid.py +++ b/lib/ants/cli/ancil_vertical_regrid.py @@ -25,8 +25,9 @@ def load_data( target_grid, begin=None, end=None, + ignore_metadata_read=None, ): - source_cubes = ants.io.load.load(source) + source_cubes = ants.io.load.load(source, ignore_metadata_read=ignore_metadata_read) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) @@ -87,6 +88,8 @@ def main( end, save_ukca, netcdf_only, + ignore_metadata_read, + ignore_metadata_write, ): """ Vertical regrid application top level call function. @@ -112,6 +115,11 @@ def main( end : :obj:`datetime`, optional If provided, all source data after this year is discarded. Default is to include all source data. + ignore_metadata_read : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. + ignore_metadata_write : bool + If True, do not write metadata sidecar files alongside the ancillary. Returns @@ -121,10 +129,7 @@ def main( """ source_cubes, target_cube = load_data( - source_path, - target_path, - begin, - end, + source_path, target_path, begin, end, ignore_metadata_read ) regridded_cubes = regrid(source_cubes, target_cube) @@ -133,7 +138,11 @@ def main( save.ukca_netcdf(regridded_cubes, output_path) else: if not netcdf_only: - save.ancil(regridded_cubes, output_path) + save.ancil( + regridded_cubes, + output_path, + ignore_metadata_write=ignore_metadata_write, + ) save.netcdf(regridded_cubes, output_path) return regridded_cubes @@ -163,6 +172,8 @@ def cli_interface(): args.end, args.save_ukca, args.netcdf_only, + args.ignore_metadata_read, + args.ignore_metadata_write, ) diff --git a/lib/ants/command_parse.py b/lib/ants/command_parse.py index 2a3a01c..30471b7 100644 --- a/lib/ants/command_parse.py +++ b/lib/ants/command_parse.py @@ -33,6 +33,14 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): sources ... Source data path(s). --output , -o Output filepath --ants-config Configuration path. + --ignore-metadata-read Stops the application from + searching for metadata files + with the same name as the + source and loading them + alongside the source. + --ignore-metadata-write Stops the application from + writing metadata files + alongside the outputs. Additionally, there are standardised optional arguments and these are activated by passing the relevant keyword argument. See 'Parameters' @@ -89,8 +97,6 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): help="Source data path(s).", ) - target_obj = self - # Arguments change below depending on whether target_lsm, target_grid, # or both are requested. land-threshold is needed with target_lsm # whether or not target_grid is asked for: @@ -141,14 +147,14 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): self.add_argument( "--target-lsm", type=ants.config.filepath_readable, - required=target_obj is self, + required=True, help="Path to the land sea mask.", ) elif target_grid: - target_obj.add_argument( + self.add_argument( "--target-grid", type=ants.config.filepath_readable, - required=target_obj is self, + required=True, nargs="+", help="Path to the target grid file.", ) @@ -171,6 +177,19 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): help="Only write out a netCDF file.", required=False, ) + self.add_argument( + "--ignore-metadata-read", + action="store_true", + help="Stops the application from searching for metadata files with " + "the same name as the source and loading them alongside the source.", + required=False, + ) + self.add_argument( + "--ignore-metadata-write", + action="store_true", + help="Stops the application from writing metadata files alongside outputs.", + required=False, + ) if time_constraints: self.add_argument( "--begin", diff --git a/lib/ants/exceptions.py b/lib/ants/exceptions.py index a2b62c3..2083933 100644 --- a/lib/ants/exceptions.py +++ b/lib/ants/exceptions.py @@ -176,3 +176,7 @@ def __init__( ) self.message = message super().__init__(self.message) + + +class MetadataWarning(Warning): + """Raised when unexpected or non-standard metadata is encountered.""" diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 9c53786..b91e50a 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -56,6 +56,8 @@ """ import copy +import glob +import inspect import warnings from contextlib import contextmanager from functools import wraps @@ -210,7 +212,7 @@ def ants_format_agent(): iris.fileformats.FORMAT_AGENT = original_format_agent -def load_landsea_mask(filename, land_threshold=None): +def load_landsea_mask(filename, land_threshold=None, ignore_metadata_read=True): """ Load a landsea mask from either a landsea mask file or a landfraction file. @@ -222,6 +224,9 @@ def load_landsea_mask(filename, land_threshold=None): Threshold for converting the land fraction field into a landsea mask field. 0.5 would mean that any fraction greater than this will be masked. This argument is used when loading a land fraction field. + ignore_metadata_read : bool + The option to turn off the ability to load external metadata + files. Set to True (off) by default. Returns ------- @@ -231,18 +236,26 @@ def load_landsea_mask(filename, land_threshold=None): """ try: # Is it a landsea mask field? - lbm = ants.io.load.load_cube(filename, "land_binary_mask") + lbm = ants.io.load.load_cube( + filename, "land_binary_mask", ignore_metadata_read=ignore_metadata_read + ) lbm = lbm.copy(lbm.data.astype("bool", copy=False)) except iris.exceptions.ConstraintMismatchError: try: # Is it a land fraction field? - land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction") + land_fraction = ants.io.load.load_cube( + filename, + "vegetation_area_fraction", + ignore_metadata_read=ignore_metadata_read, + ) lbm = land_fraction.copy(land_fraction.data > land_threshold) lbm.rename("land_binary_mask") except iris.exceptions.ConstraintMismatchError: # It looks like we are wanting to extract a landsea mask from some # other field. - cube = ants.io.load.load(filename)[0] + cube = ants.io.load.load( + filename, ignore_metadata_read=ignore_metadata_read + )[0] y = cube.coord(axis="y") x = cube.coord(axis="x") cube = cube.slices((y, x)).next() @@ -250,7 +263,7 @@ def load_landsea_mask(filename, land_threshold=None): return lbm -def load_grid(filenames, *args, **kwargs): +def load_grid(filenames, *args, ignore_metadata_read=False, **kwargs): """ Load a grid definition and return an iris cube. @@ -271,6 +284,9 @@ def load_grid(filenames, *args, **kwargs): definition. *args : See :func:`iris.load` + ignore_metadata_read : bool + The option to turn off the ability to load external metadata + files. Set to False (on) by default. **kwargs : See :func:`iris.load` @@ -298,7 +314,9 @@ def load_grid(filenames, *args, **kwargs): # for variable resolution namelists - need both parts (regular and # variable namelists) in the namelist loader at the same time. if filenames: - cubes = load(filenames, *args, **kwargs) + cubes = load( + filenames, *args, ignore_metadata_read=ignore_metadata_read, **kwargs + ) if cubes: results.extend(cubes) @@ -355,9 +373,25 @@ def load_function(*args, **kwargs): "iris.FUTURE.datum_support flag.", FutureWarning, ) - # Use context manager to avoid permanently modifying iris behaviour. - with ants_format_agent(): - cubes = func(*args, **kwargs) + # Check if metadata functionality has been turned off. + ignore_metadata_read = False + if "ignore_metadata_read" in kwargs: + ignore_metadata_read = kwargs.pop("ignore_metadata_read") + if not ignore_metadata_read: + # Get the inputs of the function. + # Chain any user callback after the metadata callback. + bound = inspect.signature(func).bind(*args, **kwargs) + bound.apply_defaults() + bound.arguments["callback"] = _CallbackMetadata( + bound.arguments["callback"] + ) + # Use context manager to avoid permanently modifying iris behaviour. + with ants_format_agent(): + cubes = func(*bound.args, **bound.kwargs) + else: + # Use context manager to avoid permanently modifying iris behaviour. + with ants_format_agent(): + cubes = func(*args, **kwargs) if cubes is not None: try: ants.utils.cube.derive_circular_status(cubes) @@ -370,7 +404,80 @@ def load_function(*args, **kwargs): return load_function -def load_cube(*args, **kwargs): +class _CallbackMetadata: + """Callback for collecting metadata from sidecar files. + + This callback will load additional metadata files with the naming convention: + filename.[license,attribution,restrictions] and append the contents of those files + to the cube attributes. + """ + + def __init__(self, user_callback): + self._user_callback = user_callback + + def __call__(self, cube, field, filename): + """ + The method that runs when Iris runs the callback. Collects the filenames and + will run the user callback + """ + metadata_filenames = f"{filename}.*" + metadata_files = glob.glob(metadata_filenames) + if metadata_files != []: + self._retrieve_metadata(metadata_files, cube) + if self._user_callback is not None: + self._user_callback(cube, field, filename) + + def _retrieve_metadata(self, metadata_files, cube): + """ + Reads in the contents of each file and adds an attribute to the cube with the + name of the file. + + Parameters + ---------- + metadata_files: list + The list of filenames that contain metadata for the cube. + cube : :class:`iris.cube.Cube` + The cube being loaded. + + """ + valid_metadata_names = [ + "license", + "attribution", + "restrictions", + "institution", + "acknowledgement", + "references", + ] + for metadata_file in metadata_files: + file_name_splits = str(metadata_file).split(".") + attribute_name = file_name_splits[-1] + # Check for misspelling + if attribute_name == "licence": + warnings.warn( + "The attribute name 'licence' has been changed to " + "'license', in line with ANTS working practices.", + category=ants.exceptions.MetadataWarning, + ) + attribute_name = "license" + if attribute_name in cube.attributes: + raise AttributeError( + f"The {attribute_name} is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." + ) + if attribute_name not in valid_metadata_names: + warnings.warn( + f"Attribute {attribute_name} is not a valid metadata file " + f"name. Accepted metadata names are: {valid_metadata_names}", + category=ants.exceptions.MetadataWarning, + ) + else: + with open(metadata_file, "r") as open_file: + metadata = open_file.read() + cube.attributes[attribute_name] = metadata + + +def load_cube(*args, ignore_metadata_read=False, **kwargs): """ Loads a single cube. @@ -382,6 +489,9 @@ def load_cube(*args, **kwargs): ---------- *args : See :func:`iris.load_cube` + ignore_metadata_read : bool + The option to turn off the ability to load external metadata + files. Set to False (on) by default. **kwargs : See :func:`iris.load_cube` @@ -390,10 +500,10 @@ def load_cube(*args, **kwargs): A :class:`~iris.cube.Cube`. """ loading_function = _customised_load(iris.load_cube) - return loading_function(*args, **kwargs) + return loading_function(*args, ignore_metadata_read=ignore_metadata_read, **kwargs) -def load(*args, **kwargs): +def load(*args, ignore_metadata_read=False, **kwargs): """ Loads any number of Cubes for each constraint. @@ -406,6 +516,9 @@ def load(*args, **kwargs): ---------- *args : See :func:`iris.load` + ignore_metadata_read : bool + The option to turn off the ability to load external metadata + files. Set to False (on) by default. **kwargs : See :func:`iris.load` @@ -414,10 +527,10 @@ def load(*args, **kwargs): A :class:`~iris.cube.CubeList`. """ loading_function = _customised_load(iris.load) - return loading_function(*args, **kwargs) + return loading_function(*args, ignore_metadata_read=ignore_metadata_read, **kwargs) -def load_cubes(*args, **kwargs): +def load_cubes(*args, ignore_metadata_read=False, **kwargs): """ Loads exactly one Cube for each constraint. @@ -430,6 +543,9 @@ def load_cubes(*args, **kwargs): ---------- *args : See :func:`iris.load_cubes` + ignore_metadata_read : bool + The option to turn off the ability to load external metadata + files. Set to False (on) by default. **kwargs : See :func:`iris.load_cubes` @@ -439,10 +555,10 @@ def load_cubes(*args, **kwargs): """ loading_function = _customised_load(iris.load_cubes) - return loading_function(*args, **kwargs) + return loading_function(*args, ignore_metadata_read=ignore_metadata_read, **kwargs) -def load_raw(*args, **kwargs): +def load_raw(*args, ignore_metadata_read=False, **kwargs): """ Loads non-merged cubes. @@ -455,6 +571,9 @@ def load_raw(*args, **kwargs): ---------- *args : See :func:`iris.load_raw` + ignore_metadata_read : bool + The option to turn off the ability to load external metadata + files. Set to False (on) by default. **kwargs : See :func:`iris.load_raw` @@ -463,4 +582,4 @@ def load_raw(*args, **kwargs): A :class:`~iris.cube.CubeList`. """ loading_function = _customised_load(iris.load_raw) - return loading_function(*args, **kwargs) + return loading_function(*args, ignore_metadata_read=ignore_metadata_read, **kwargs) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 9ebd7bf..3f1ccd7 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -20,12 +20,14 @@ specifying ``saver='ukca'`` (see :func:`ants.io.save.ukca_netcdf`). """ +import logging import os import sys import warnings import ants.utils.cube import iris +import numpy as np from ants.fileformats.ancil import _cubes_to_ancilfile, _mule_set_lbuser2 from ants.fileformats.netcdf.cf import ( _coerce_netcdf_classic_dtypes, @@ -34,7 +36,7 @@ from ants.fileformats.netcdf.ukca import LOCAL_ATTS, _ukca_conventions -def ancil(cubes, filename): +def ancil(cubes, filename, ignore_metadata_write=False): """ Save one or more cubes to a F03 UM ancillary file. @@ -72,6 +74,9 @@ def ancil(cubes, filename): One or more cubes to be saved. filename : str The name of the F03 UM ancillary file, including any extension. + ignore_metadata_write : bool + Determines whether attributes should be saved to a seperate metadata file. + Default setting is False, so will write out the metadata. Notes ----- @@ -91,6 +96,8 @@ def ancil(cubes, filename): raise ValueError("F03 UM ancillary files cannot be saved with a .nc extension.") cubes = ants.utils.cube.as_cubelist(cubes) + if not ignore_metadata_write: + _check_and_sort_metadata_attributes(cubes, filename) ancilfile = _cubes_to_ancilfile(cubes) _mule_set_lbuser2(ancilfile) ancilfile.to_file(filename) @@ -319,3 +326,88 @@ def _update_history_cmd(cube): items[0] = os.path.basename(items[0]) items.append(f"({metadata})") if metadata else None ants.utils.cube.update_history(cubes, " ".join(items)) + + +def _check_and_sort_metadata_attributes(cubes, data_filepath): + """Checks for a license, attribution or restrictions in the metadata of the cubes, + and calls `_write_metadata_file` for each attribute. + Parameters + ---------- + cubes : :class:`iris.cube.Cube` or :class:`iris.cube.CubeList` + One or more cubes to be saved. + data_filepath : str + The name of the file where the data will be saved to. + """ + # a dictionary to contain all of the metadata to be saved + metadata_dictionary = {} + # a dictionary to keep track of which cubes have metadata + cube_names_dictionary = {} + # a list of approved attributes that can be saved + attributes_to_save = [ + "license", + "attribution", + "restrictions", + "institution", + "acknowledgement", + "references", + ] + for cube in cubes: + for key, value in cube.attributes.items(): + # check the attribute is one we want to save + if key == "licence": + warnings.warn( + "The attribute name licence has been changed to " + "'license', in line with ANTS working practices.", + category=UserWarning, + ) + key = "license" + if key in attributes_to_save: + if key in metadata_dictionary: + metadata_dictionary[key].append(value) + cube_names_dictionary[key + "_names"].append(cube.name()) + else: + metadata_dictionary[key] = [value] + cube_names_dictionary[key + "_names"] = [cube.name()] + for key, value in metadata_dictionary.items(): + # Update the metadata ready to save + metadata_dictionary[key] = _check_multiple_attributes( + metadata_dictionary[key], cube_names_dictionary[key + "_names"] + ) + # write the metadata + _write_metadata_file(metadata_dictionary[key], data_filepath, key) + + +def _check_multiple_attributes(attribute_list, cube_names): + """Checks whether the attribute can be written out exactly as is, or if it has to be + pre-pended with the cube name.""" + # check if multiple things in list + if len(attribute_list) == 1: + return attribute_list + # check if attributes are the same + if len(set(attribute_list)) == 1: + return attribute_list[:1] + # check if there is only one cube + if len(cube_names) == 1: + return attribute_list + # if they are not the same, add the cube name + concatenated_attribute = [] + for attribute, name in zip(attribute_list, cube_names, strict=True): + concatenated_attribute.append(name + " = " + attribute + "\n") + return concatenated_attribute + + +def _write_metadata_file(metadata, filename, attribute_name): + """Takes a list of metadata and writes it to a file called + filename.. + If for any reason, the file to be written already exists, the new metadata will be + appended to it. + """ + filepath = str(filename) + "." + attribute_name + # Order metadata to be in one list, if metadata contains list of lists - possible in + # cases where metadata is being read in + if any(isinstance(element, list) for element in metadata): + metadata = np.concatenate(metadata).tolist() + with open(filepath, "a") as metadata_file: + metadata_file.writelines(metadata) + _LOGGER = logging.getLogger(__name__) + _LOGGER.info(f"{attribute_name} has been written to sidecar file {filepath}") diff --git a/lib/ants/tests/__init__.py b/lib/ants/tests/__init__.py index 1836ffc..343c70e 100644 --- a/lib/ants/tests/__init__.py +++ b/lib/ants/tests/__init__.py @@ -2,9 +2,14 @@ # # This file is part of ANTS and is released under the BSD 3-Clause license. # See LICENSE.txt in the root of the repository for full licensing details. +# Some of the content of this file has been produced with the assistance of +# Met Office GitHub Copilot Enterprise. import collections +import contextlib import functools import os +import shutil +import tempfile import unittest import warnings @@ -146,6 +151,37 @@ def get_data_path(relative_path): return os.path.abspath(os.path.join(_RESOURCE_PATH, relative_path)) +@contextlib.contextmanager +def isolated_resource(relative_path, sidecars=None): + """ + Copy a resource file into a standalone temporary directory. + The path is relative to the directory returned by `get_data_path`. The + path of the copied file is yielded, and the temporary directory is removed + on exit. Useful for testing sidecar metadata file loading, where + neighbouring files matter. + Parameters + ---------- + relative_path : str + Resource file to copy. + sidecars : dict, optional + Maps sidecar extensions to text content, for files to create alongside + the copied file, named ``.``. + Example usage: + def test_sidecar(self): + with isolated_resource( + "global_geodetic.tif", sidecars={"license": "A license\\n"} + ) as path: + cube = ants.load_cube(path) + """ + with tempfile.TemporaryDirectory(prefix="ants_test_resources_") as tmp: + path = os.path.join(tmp, os.path.basename(relative_path)) + shutil.copy2(get_data_path(relative_path), path) + for extension, content in (sidecars or {}).items(): + with open(f"{path}.{extension}", "x") as sidecar: + sidecar.write(content) + yield path + + class TestCase(unittest.TestCase): _assertion_counts = collections.defaultdict(int) diff --git a/lib/ants/tests/command_parse/test_integration.py b/lib/ants/tests/command_parse/test_integration.py index 31105ab..a910dd4 100644 --- a/lib/ants/tests/command_parse/test_integration.py +++ b/lib/ants/tests/command_parse/test_integration.py @@ -30,6 +30,7 @@ def setUp(self): self.addCleanup(patch.stop) def test_default_args(self): + """Tests that the default arguments are as expected.""" new = [ "program", "/path/to/source", @@ -53,6 +54,8 @@ def path_check(filename, **kwargs): output="/path/to/output", sources=["/path/to/source"], netcdf_only=False, + ignore_metadata_read=False, + ignore_metadata_write=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -69,6 +72,8 @@ def test_add_args(self): output="/path/to/output", ants_config=None, netcdf_only=False, + ignore_metadata_read=False, + ignore_metadata_write=False, new_arg="new_arg_value", ) self.assertEqual(args, target_args) @@ -150,6 +155,7 @@ def test_invalid_output_same_file(self): self.mock_dirpath_writeable.assert_called_once() def test_time_constraint_args(self): + """Tests that using time constraint arguments will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -176,11 +182,14 @@ def test_time_constraint_args(self): begin=2016, end=2021, netcdf_only=False, + ignore_metadata_read=False, + ignore_metadata_write=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) def test_time_constraint_flags(self): + """Tests that using time constraint flags will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -207,6 +216,8 @@ def test_time_constraint_flags(self): begin=1990, end=1996, netcdf_only=False, + ignore_metadata_read=False, + ignore_metadata_write=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -265,3 +276,59 @@ def test_both_time_constraints_exist(self): parser = AntsArgParser(target_lsm=True, time_constraints=True) with self.assertRaises(exceptions.TimeConstraintMissingException): parser.parse_args() + + def test_set_ignore_metadata_read_flag(self): + "Tests that ignore_metadata_read will be set to True when passed in." + new = [ + "program", + "/path/to/source", + "--target-lsm", + "/path/to/lsm", + "-o", + "/path/to/output", + "--ignore-metadata-read", + ] + with mock.patch("sys.argv", new=new): + parser = AntsArgParser(target_lsm=True) + args = parser.parse_args() + + target_args = argparse.Namespace( + ants_config=None, + land_threshold=None, + target_lsm="/path/to/lsm", + output="/path/to/output", + sources=["/path/to/source"], + netcdf_only=False, + ignore_metadata_read=True, + ignore_metadata_write=False, + ) + self.assertFalse(self.mock_config.called) + self.assertEqual(args, target_args) + + def test_set_ignore_metadata_write_flag(self): + "Tests that ignore_metadata_write will be set to True when passed in." + new = [ + "program", + "/path/to/source", + "--target-lsm", + "/path/to/lsm", + "-o", + "/path/to/output", + "--ignore-metadata-write", + ] + with mock.patch("sys.argv", new=new): + parser = AntsArgParser(target_lsm=True) + args = parser.parse_args() + + target_args = argparse.Namespace( + ants_config=None, + land_threshold=None, + target_lsm="/path/to/lsm", + output="/path/to/output", + sources=["/path/to/source"], + netcdf_only=False, + ignore_metadata_read=False, + ignore_metadata_write=True, + ) + self.assertFalse(self.mock_config.called) + self.assertEqual(args, target_args) diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py new file mode 100644 index 0000000..9f406af --- /dev/null +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -0,0 +1,185 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. +""" +Includes tests for end-to-end functionality of CallbackMetadata as well as calling the +class directly. +""" + +import unittest.mock as mock +import warnings + +import ants.io.load +import iris +import pytest + + +def test_metadata_files_added_to_attributes(tmp_path): + """Tests that metadata files are found and added to the cube's attributes.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + # How the text should look while stored in an array + loaded_license = ( + "This is the license of the cube.\n\n It should be preserved" + " and added to the cube when loaded.\n " + ) + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_test_cube.attributes["license"] == loaded_license + + +def test_no_metadata_loaded(tmp_path): + """Tests that metadata files are not loaded when the option is turned off.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube( + temporary_cube_path, ignore_metadata_read=True + ) + assert "license" not in loaded_test_cube.attributes + + +def test_user_callback_added(): + """Test that on ititialisation, the user's function will be set.""" + + def user_callback(cube, field, filename): + print("a user's callback, passed in") + + class_instance = ants.io.load._CallbackMetadata(user_callback) + assert class_instance._user_callback == user_callback + + +def test_user_callback_called__keyword_arg(tmp_path): + """Tests that when passed a user callback using a keyword argument, + the callback is parsed correctly.""" + mock_callback = mock.Mock() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + ants.io.load.load_cube(temporary_cube_path, callback=mock_callback) + mock_callback.assert_called() + + +def test_user_callback_called__positional_arg(tmp_path): + """Tests that when passed a user callback using a keyword argument, + the callback is parsed correctly.""" + mock_callback = mock.Mock() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + ants.io.load.load_cube(temporary_cube_path, None, mock_callback) + mock_callback.assert_called() + + +def test__retrieve_metadata_correct_name(): + """Tests that correctly named metadata files are going to be read.""" + class_instance = ants.io.load._CallbackMetadata(None) + mock_file_open = mock.mock_open() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + path = ["fake-path/fake-cube.attribution"] + with mock.patch("builtins.open", mock_file_open): + class_instance._retrieve_metadata(path, test_cube) + mock_file_open.assert_called_once_with("fake-path/fake-cube.attribution", "r") + + +def test__retrieve_metadata_incorrect_name(): + """Tests that files without a valid metadata name won't be read.""" + class_instance = ants.io.load._CallbackMetadata(None) + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + path = ["fake-path/fake-cube.invalid"] + expected_message = ( + "Attribute invalid is not a valid metadata file " + r"name. Accepted metadata names are: \['license', 'attribution', " + r"'restrictions', 'institution', 'acknowledgement', 'references'\]" + ) + with pytest.raises(ants.exceptions.MetadataWarning, match=expected_message): + class_instance._retrieve_metadata(path, test_cube) + + +def test_attribute_already_on_cube(): + """Tests that having both metadata attributes and sidecar files of the same type + will raise an attribute error.""" + class_instance = ants.io.load._CallbackMetadata(None) + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + test_cube.attributes["attribution"] = "This is a attribution. " + path = ["fake-path/fake-cube.attribution"] + expected_message = ( + "The attribution is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." + ) + with pytest.raises(AttributeError, match=expected_message): + class_instance._retrieve_metadata(path, test_cube) + + +@pytest.mark.filterwarnings( + "ignore:The attribute name 'licence' has been changed to 'license', in line with " + "ANTS working practices.:ants.exceptions.MetadataWarning" +) +def test_misspelt_license_with_licensed_cube(): + """Tests that when given a sidecar file with a miss-spelt license and a cube with an + existing license, an error will be given.""" + class_instance = ants.io.load._CallbackMetadata(None) + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + test_cube.attributes["license"] = "This is a license. " + path = ["fake-path/fake-cube.licence"] + expected_message = ( + "The license is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." + ) + with pytest.raises(AttributeError, match=expected_message): + class_instance._retrieve_metadata(path, test_cube) + + +def test_misspelt_license_added(tmp_path): + """Tests that a different spelling of license will add a license attribute.""" + license_text = "a license" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.licence" + temporary_license_path.write_text(license_text, encoding="utf-8") + # ignore warning that will be raised + warning_message = ( + "The attribute name 'licence' has been changed to 'license', in " + "line with ANTS working practices." + ) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=warning_message, category=ants.exceptions.MetadataWarning + ) + loaded_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_cube.attributes["license"] == "a license" + + +def test_invalid_metadata_name(tmp_path): + """Tests that an invalid metadata name will not be added as an attribute.""" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.invalid-name" + temporary_license_path.write_text(" ", encoding="utf-8") + + warning_message = ( + "Attribute invalid-name is not a valid metadata file " + r"name. Accepted metadata names are: \['license', 'attribution', " + r"'restrictions', 'institution', 'acknowledgement', 'references'\]" + ) + with pytest.raises(ants.exceptions.MetadataWarning, match=warning_message): + ants.io.load.load_cube(temporary_cube_path) diff --git a/lib/ants/tests/io/load/test_ancil.py b/lib/ants/tests/io/load/test_ancil.py index 9e786ff..a998d1f 100644 --- a/lib/ants/tests/io/load/test_ancil.py +++ b/lib/ants/tests/io/load/test_ancil.py @@ -2,12 +2,15 @@ # # This file is part of ANTS and is released under the BSD 3-Clause license. # See LICENSE.txt in the root of the repository for full licensing details. +# Some of the content of this file has been produced with the assistance of +# Met Office GitHub Copilot Enterprise. import tempfile import ants import ants.tests import iris.fileformats +import pytest from ants.fileformats.ancil import mule from ants.io.load import ants_format_agent @@ -103,3 +106,94 @@ def test_forecast_reference_time(): result = ants.io.load.load_cube(temp_file.name) result_coords = [c.name() for c in result.coords()] assert "forecast_reference_time" not in result_coords + + +@ants.tests.skip_mule +def test_metadata_loaded(): + """Loads an ancil file, along with associated metadata from sidecar files.""" + expected_attributes = { + "license": "An example license\n", + "attribution": "An example attribution\n", + "restrictions": "An example restrictions\n", + "institution": "An example institution\n", + "acknowledgement": "An example acknowledgement\n", + "references": "An example references\n", + } + + with ants.tests.isolated_resource( + "load_files/ancil_file_with_pseudo_levels", sidecars=expected_attributes + ) as file_path: + loaded_cube = ants.io.load.load_cube(file_path) + + assert loaded_cube.attributes["license"] == "An example license\n" + assert loaded_cube.attributes["attribution"] == "An example attribution\n" + assert loaded_cube.attributes["restrictions"] == "An example restrictions\n" + assert loaded_cube.attributes["institution"] == "An example institution\n" + assert loaded_cube.attributes["acknowledgement"] == "An example acknowledgement\n" + assert loaded_cube.attributes["references"] == "An example references\n" + + +@ants.tests.skip_mule +def test_metadata_not_loaded(): + """Loads an ancil file, ignoring associated metadata from sidecar files.""" + with ants.tests.isolated_resource( + "load_files/ancil_file_with_pseudo_levels", + sidecars={"license": "An example license\n"}, + ) as file_path: + loaded_cube = ants.io.load.load_cube(file_path, ignore_metadata_read=True) + assert "license" not in loaded_cube.attributes + + +@ants.tests.skip_mule +# pytest.warns re-emits unmatched warnings with module "warnings", so the +# module-scoped "mule" filter in pyproject.toml no longer applies. +@pytest.mark.filterwarnings( + "ignore:Ancillary files do not define the UM version number:UserWarning" +) +@pytest.mark.filterwarnings( + "ignore:`product` is deprecated as of NumPy 1.25.0, and will be removed in NumPy 2.0. Please use `prod` instead.:DeprecationWarning" # noqa: E501 +) +def test_misnamed_licence_warning(): + """Loads an ancil file, along with associated metadata from licence sidecar file. + If a sidecar file has the .licence extension, a warning will be raised, and + the metadata should be added to the license attribute. + """ + expected_msg = ( + "The attribute name 'licence' has been changed to " + "'license', in line with ANTS working practices." + ) + with ants.tests.isolated_resource( + "load_files/ancil_file_with_pseudo_levels", + sidecars={"licence": "An example licence\n"}, + ) as file_path: + with pytest.warns(ants.exceptions.MetadataWarning, match=expected_msg): + loaded_cube = ants.io.load.load_cube(file_path) + assert loaded_cube.attributes["license"] == "An example licence\n" + + +@ants.tests.skip_mule +# pytest.warns re-emits unmatched warnings with module "warnings", so the +# module-scoped "mule" filter in pyproject.toml no longer applies. +@pytest.mark.filterwarnings( + "ignore:Ancillary files do not define the UM version number:UserWarning" +) +@pytest.mark.filterwarnings( + "ignore:`product` is deprecated as of NumPy 1.25.0, and will be removed in NumPy 2.0. Please use `prod` instead.:DeprecationWarning" # noqa: E501 +) +def test_invalid_metadata_warning(): + """Loads an ancil file, and tests that a warning is raised for an unsupported + sidecar file: .invalid""" + expected_msg = ( + "Attribute invalid is not a valid metadata file " + r"name. Accepted metadata names are: \['license', 'attribution', " + r"'restrictions', 'institution', 'acknowledgement', 'references'\]" + ) + + with ants.tests.isolated_resource( + "load_files/ancil_file_with_pseudo_levels", + sidecars={"invalid": "Invalid metadata file content"}, + ) as file_path: + with pytest.warns(ants.exceptions.MetadataWarning, match=expected_msg): + loaded_cube = ants.io.load.load_cube(file_path) + + assert "invalid" not in loaded_cube.attributes diff --git a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py new file mode 100644 index 0000000..e2190a5 --- /dev/null +++ b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py @@ -0,0 +1,140 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. +import logging +from unittest import mock + +import ants.tests.stock as stock +import pytest +from ants.io.save import _check_and_sort_metadata_attributes + + +def test_license_attribute_written(tmp_path): + """Tests that a cube with a license is written out.""" + cube = stock.geodetic(shape=(2, 2)) + license = "This is a cube's license. " + cube.attributes["license"] = license + cube.rename("license test cube") + filename = tmp_path / "test_cube" + _check_and_sort_metadata_attributes([cube], filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.read() + assert actual_license == license + + +@pytest.mark.filterwarnings( + "ignore:The attribute name licence has been changed to 'license', in line with " + "ANTS working practices.:UserWarning" +) +def test_licence_attribute_written(tmp_path): + """Tests that a cube with a licence is changed and written out.""" + cube = stock.geodetic(shape=(2, 2)) + licence = "This is a cube's license. " + cube.attributes["licence"] = licence + cube.rename("licence test cube") + filename = tmp_path / "test_cube" + _check_and_sort_metadata_attributes([cube], filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.read() + assert actual_license == licence + + +def test_loaded_license_written(tmp_path): + """Tests that a cube with a license that resembles the format of a longer license + written in, is written out correctly.""" + loaded_license = [ + "This is the license of the cube.\n", + "\n", + " It should be preserved and added to the cube when loaded.\n", + " ", + ] + cube = stock.geodetic(shape=(2, 2)) + cube.attributes["license"] = loaded_license + cube.rename("loaded license test cube") + filename = tmp_path / "test_cube" + _check_and_sort_metadata_attributes([cube], filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.readlines() + assert actual_license == loaded_license + + +def test_log_output(caplog): + """Tests that the logger gives the correct output when a file is written out.""" + cube = stock.geodetic(shape=(2, 2)) + institution = "This data came from University Blah. " + cube.attributes["institution"] = institution + expected_message = ( + "institution has been written to sidecar file filename.institution" + ) + # mocks out the opening of files, so no file is created + with mock.patch("builtins.open"): + with caplog.at_level(logging.INFO): + _check_and_sort_metadata_attributes([cube], "filename") + assert expected_message in caplog.text + + +def test_multiple_cubes(tmp_path): + """Test that multiple_cubes with attributes is written out correctly.""" + # creating three cubes with different license attributes + cube1 = stock.geodetic(shape=(2, 2)) + license1 = "This is a cube's license. " + cube1.attributes["license"] = license1 + cube1.rename("the first cube") + cube2 = stock.geodetic(shape=(2, 2)) + license2 = "This is another cube's license. " + cube2.attributes["license"] = license2 + cube2.rename("the second cube") + cube3 = stock.geodetic(shape=(2, 2)) + license3 = "This is a third cube's license. " + cube3.attributes["license"] = license3 + cube3.rename("the third cube") + cubelist = [cube1, cube2, cube3] + filename = tmp_path / "multiple_cube_test" + # The actual test + _check_and_sort_metadata_attributes(cubelist, filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.read() + expected_license = ( + "the first cube = This is a cube's license. \nthe second cube = " + "This is another cube's license. \nthe third cube = This is a third cube's " + "license. \n" + ) + assert actual_license == expected_license + + +def test_all_different_attributes_written_out(): + """Tests that a cube with a multiple different attributes writes + out all metadata files.""" + cube = stock.geodetic(shape=(2, 2)) + attribution = "This data came from an institution. " + cube.attributes["attribution"] = attribution + cube.attributes["restrictions"] = ( + "This data is restricted to be used for testing purposes only." + ) + cube.attributes["license"] = "This is a license for the data" + filename = "test_multiple_attributes" + with mock.patch("ants.io.save._write_metadata_file") as mock_method: + _check_and_sort_metadata_attributes([cube], filename) + + expected_license = mock.call( + ["This is a license for the data"], "test_multiple_attributes", "license" + ) + expected_attribution = mock.call( + ["This data came from an institution. "], + "test_multiple_attributes", + "attribution", + ) + expected_restrictions = mock.call( + ["This data is restricted to be used for testing purposes only."], + "test_multiple_attributes", + "restrictions", + ) + + assert expected_license in mock_method.call_args_list + assert expected_attribution in mock_method.call_args_list + assert expected_restrictions in mock_method.call_args_list diff --git a/lib/ants/tests/io/save/test__check_multiple_attributes.py b/lib/ants/tests/io/save/test__check_multiple_attributes.py new file mode 100644 index 0000000..1d37924 --- /dev/null +++ b/lib/ants/tests/io/save/test__check_multiple_attributes.py @@ -0,0 +1,47 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. + +from ants.io.save import _check_multiple_attributes + + +def test_one_element_in_list(): + """Tests that when given a list with one element, that is returned.""" + expected = ["This is the only attribution given. "] + cube_name = ["a cube with an attribution"] + actual = _check_multiple_attributes(expected, cube_name) + assert expected == actual + + +def test_only_one_Cube_in_list(): + """Tests that when given only one cube name, the attributes are returned.""" + expected = [ + "This is an attribution for some of the data. ", + "This is a different attribution for the rest of the data.", + ] + cube_name = ["a cube with multiple attributions"] + actual = _check_multiple_attributes(expected, cube_name) + assert expected == actual + + +def test_all_elements_in_list_same(): + """Tests that when all elements in list are the same, one is returned.""" + expected = ["This is the only attribution given. "] + cube_name = ["a cube with an attribution"] + attribute_list = [ + "This is the only attribution given. ", + "This is the only attribution given. ", + "This is the only attribution given. ", + ] + actual = _check_multiple_attributes(attribute_list, cube_name) + assert expected == actual + + +def test_different_elements_have_cube_names(): + """Tests that when multiple attributes are given, cube names are included.""" + expected = ["cube1 = license 1. \n", "cube2 = license 2. \n"] + licenses = ["license 1. ", "license 2. "] + cube_names = ["cube1", "cube2"] + actual = _check_multiple_attributes(licenses, cube_names) + assert expected == actual diff --git a/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py new file mode 100644 index 0000000..f41feb1 --- /dev/null +++ b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py @@ -0,0 +1,108 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. + +from ants.tests import stock +from ants.utils.cube import copy_metadata_attributes + + +def test_adding_new_attribute(): + """Tests that a new attribute will be added to the source cube if it doesn't exist + already.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + # Reference cube: + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["license"] = "a test license" + reference.rename("test cube") + # Copy the metadata + copy_metadata_attributes(source, reference) + assert source.attributes["license"] == "test cube = a test license" + + +def test_modifying_existing_attribute(): + """Tests that an existing attribute will be modified on the source cube.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["references"] = "a test reference" + source.rename("source cube") + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["references"] = "another different test reference" + reference.rename("reference cube") + # Copy the metadata + copy_metadata_attributes(source, reference) + assert ( + source.attributes["references"] + == "source cube = a test reference\n" + + "reference cube = another different test reference" + ) + + +def test_no_reference_attribute(): + """Tests that a source cube attribute with no reference will be left unchanged.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["restrictions"] = "a test restriction" + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + # Copy the metadata + copy_metadata_attributes(source, reference) + assert source.attributes["restrictions"] == "a test restriction" + + +def test_non_standard_attribute(): + """Tests that an attribute not in the allowed list is not copied over.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + # Reference cube: + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["a value not in the allowed list"] = "not in the list" + reference.rename("test cube") + # Copy the metadata + copy_metadata_attributes(source, reference) + assert "a value not in the allowed list" not in source.attributes + + +def test_different_attribute_list(): + """Tests that a different attribute list will copy over those attributes.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + # Reference cube: + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["test-attribute"] = "an attribute not in the standard list." + reference.rename("test cube") + copy_metadata_attributes(source, reference, metadata_to_copy=["test-attribute"]) + assert ( + source.attributes["test-attribute"] + == "test cube = an attribute not in the standard list." + ) + + +def test_same_attributes(): + """Tests that if both cubes have the same attribute, nothing happens.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["restrictions"] = "a test restriction" + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["restrictions"] = "a test restriction" + # Copy the metadata + copy_metadata_attributes(source, reference) + # Check that the attribute is unchanged + assert source.attributes["restrictions"] == "a test restriction" + + +def test_different_whitespace(): + """Tests that attributes with different whitespace will be registered as equal.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["restrictions"] = "a test restriction" + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["restrictions"] = "a test restriction " + # Copy the metadata + copy_metadata_attributes(source, reference) + # Check that the attribute is unchanged + assert source.attributes["restrictions"] == "a test restriction" diff --git a/lib/ants/utils/cube.py b/lib/ants/utils/cube.py index 076e919..413a964 100644 --- a/lib/ants/utils/cube.py +++ b/lib/ants/utils/cube.py @@ -1139,6 +1139,63 @@ def inherit_metadata(source, reference): source.attributes["grid_staggering"] = reference.attributes["grid_staggering"] +def copy_metadata_attributes(source, reference, metadata_to_copy=None): + """ + Inherit cube metadata attributes from a provided reference. + + In-place operation on source cube. + + Metadata attributes from the refrence are added into the source cube. This is done + in the manner of `attribute: cube-name = metadata content` for tracebaility and + allowing for the stacking of metadata. + + Parameters + ---------- + source : :class:`iris.cube.Cube` + Source to have its metadata update. + reference : :class:`iris.cube.Cube` + Reference which defines the metadata to inherit from. + metadata_to_copy : list + A list of metadata attribute keys to copy to the source cube. + """ + if metadata_to_copy is None: + metadata_to_copy = [ + "license", + "attribution", + "restrictions", + "institution", + "acknowledgement", + "references", + ] + + for attribute in metadata_to_copy: + # Only copy an attribute if it exists in the reference cube. + if attribute in reference.attributes: + # If the attribute already exists in the source cube. + if attribute in source.attributes: + # Remove whitepace before comparisons. + source_compare = " ".join(source.attributes[attribute].split()) + reference_compare = " ".join(reference.attributes[attribute].split()) + # If the attributes are the same then nothing needs to be done. + # If the attributes are not the same then both names must be prepended + # to ensure that the metadata is traceable. + if source_compare != reference_compare: + source.attributes[attribute] = ( + source.name() + + " = " + + source.attributes[attribute] + + "\n" + + reference.name() + + " = " + + reference.attributes[attribute] + ) + else: + # If the attribute does not exist in the source cube. + source.attributes[attribute] = ( + reference.name() + " = " + reference.attributes[attribute] + ) + + def set_crs(cube, crs=None): """ Set cube coordinate system. diff --git a/pyproject.toml b/pyproject.toml index 33b059a..f5bbff8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ skip = ".git" # Configure pytest. [tool.pytest.ini_options] addopts = "--durations=5" +tmp_path_retention_policy = "none" filterwarnings = [ # During unittesting only, treat all warnings as errors by default: 'error', diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf new file mode 100644 index 0000000..9b79396 --- /dev/null +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf @@ -0,0 +1,15 @@ +[ants_decomposition] +x_split=0 +y_split=0 + +[command] +default=ants-launch ancil_general_regrid.py \ + =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ + =${begin} ${end} --search-method ${search_method} + +[env] +output=${ROSE_DATA}/${ROSE_TASK_NAME} +search_method=kdtree +source=/data/users/theo.geddes/ants-source/n96e_orca_land_cover_fraction +target=${TEST_SOURCES_DIR}/ancil_general_regrid/qrparm.mask.nc +target_type=lsm diff --git a/rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf b/rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf new file mode 100644 index 0000000..e9b1eea --- /dev/null +++ b/rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf @@ -0,0 +1,3 @@ +[env] +filelist=ancil_general_regrid_metadata + =ancil_general_regrid_metadata.nc diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index 3bcd93d..e38ca91 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -15,6 +15,7 @@ "install_graph" : "install_cold", "ancil_2anc_graph" : "install_cold => ancil_2anc => rose_ana_2anc:fail? => plot_comparisons_2anc", "fill_n_merge_land_cover_graph" : "install_cold => ancil_create_ite_shapefile => ancil_fill_n_merge_land_cover & ancil_fill_n_merge_land_cover_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", + "load_metadata_regrid_graph" : "install_cold => ancil_general_regrid_metadata => rose_ana_general_regrid", "fill_n_merge_invert_mask_graph" : "install_cold => ancil_fill_n_merge_invert_mask & ancil_fill_n_merge_invert_mask_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", "general_regrid_grid_to_grid_graph" : "install_cold => ancil_general_regrid_grid_to_grid => rose_ana_general_regrid:fail? => plot_comparisons_general_regrid install_cold => ancil_general_regrid_grid_to_grid_latitude_weighted_kdtree => rose_ana_general_regrid_latitude_weighted_kdtree:fail? => plot_comparisons_general_regrid_latitude_weighted_kdtree", @@ -258,6 +259,18 @@ fi inherit=ANCIL_GENERAL_REGRID_WITH_TIME_CONSTRAINT script = rose task-run --app-key=ancil_general_regrid_with_time_constraint -O latitude-weighted-kdtree + [[ancil_general_regrid_metadata]] + inherit = ANTS_CORE, ANCIL_GENERAL_REGRID, LARGE + script = rose task-run --app-key=ancil_general_regrid -O metadata + + [[rose_ana_general_regrid_metadata]] + inherit = ROSE_ANA + [[[environment]]] + ROSE_TASK_APP = rose_ana + ROSE_APP_OPT_CONF_KEYS = general_regrid_metadata + ANTS_KGO_DIRECTORY_OVERRIDE = /data/users/theo.geddes/ants-source/KGO/ + + [[rose_ana_general_regrid]] inherit = ROSE_ANA [[[environment]]]