Add the Ability to Load and Save External Metadata

docs/source/tutorial_load_save.rst

@@ -1,6 +1,6 @@
 .. meta::
    :description lang=en: Tutorial on loading and saving data with ANTS
-   :keywords: load, save, netcdf, ancil, tutorial
+   :keywords: load, save, netcdf, ancil, metadata, license, licensing, attribution, references, tutorial
    :property=og:locale: en_GB
 
 .. include:: common.txt
@@ -29,10 +29,13 @@ some ANTS specific ones, see :mod:`ants.fileformats`), then additionally:
    none, and
 3. removes any ``forecast_reference_time`` and ``forecast_period``
    coordinates, which are not usually wanted in ancillary fields.
+4. preserves metadata by loading sidecar metdata files alongside the data
+   files.
 
 :mod:`ants.io.save` similarly extends :py:func:`iris.save` with support for
 the F03 UM ancillary fileformat and a UKCA specific flavour of NetCDF, neither
-of which Iris can write natively.
+of which Iris can write natively, and has the option to write metadata files
+alongside the saved files in the case of saving an F03 UM ancillary fileformat.
 
 Loading data
 ------------
@@ -82,6 +85,45 @@ a land binary mask field or a land fraction field:
         "/path/to/target_lsm.nc", land_threshold=0.5
     )
 
+Loading metadata
+----------------
+Metadata in this tutorial refers to information about the data in a file, that does not
+directly affect that data. For example, licensing information or attributions.
+
+Metadata in this tutorial refers to information about the data in a file, that does not
+directly affect that data. For example, licensing information or attributions.
+
+Some ancillary file formats cannot include metadata. Metadata can be very important to
+keep alongside the data as often data will have some form of requirement or restriction
+on it. Because of this, ANTS can handle in external metadata files, provided they match
+the naming convention of `filename.extension.accepted-metadata`. The current accepted
+metadata attributes are "license", "attribution", "restrictions", "institution",
+"acknowledgement", and "references".
+
+This functionality can be turned off through the use of the argument
+``--ignore-metadata-read`` on the command line or when calling the ANTS load:
+
+.. code-block:: python
+
+    cube = ants.io.load.load("/path/to/source_file" ,ignore_metadata_read=True)
+
+
+.. note::
+    Any attributes or files referencing licensing should use the 'license' spelling for
+    consistency.
+
+ANTS can load in external metadata files or 'sidecar' files, if they are kept in the
+same directory as the data files, and have the same name (including the extension).
+Metadata can be loaded alongside all files, including NetCDF files, however ANTS will
+not allow you to load a file with a metadata attribute and a sidecar file with the same
+attribute. This is to prevent unintentional loss or overwriting of metadata information.
+
+.. note::
+    When using wildcards for loading with sidecar files, add the file extension to the
+    end to prevent issues with ANTS attempting to load the sidecar files. E.g.
+    `data*.pp` rather than `data*`.
+
+
 Saving data
 -----------
 
@@ -114,6 +156,24 @@ data type coercion and attribute renaming) on top of a standard NetCDF save:
 
     save.ukca_netcdf(source_cubes, "/path/to/output_ukca.nc")
 
+Saving metadata
+---------------
+Metadata will only be saved to a sidecar file, if the ancil saver is used. This is
+because NetCDF files will include the attributes within the file.
+
+This functionality can be turned off through the use of the argument
+``--ignore-metadata-write`` on the command line or when calling the ANTS save:
+
+.. code-block:: python
+
+    import ants.io.save as save
+
+    save.ancil(source_cubes, "/path/to/output_ancil", ignore_metadata_write=True)
+
+The sidecar files produced by ANTS follow the same naming coventions at those loaded in
+`filename.attribute.accepted-metadata`. Only accepted metadata will be written out.
+
+
 Key Points
 ----------
 

lib/ants/cli/ancil_2anc.py

@@ -37,16 +37,23 @@
 import iris
 
 
-def load_data(source):
+def load_data(source, ignore_metadata_read):
     with warnings.catch_warnings():
         warnings.filterwarnings(
             "ignore", "NetCDF default loading", iris._deprecation.IrisDeprecation
         )
-        cubes = ants.io.load.load(source)
+        cubes = ants.io.load.load(source, ignore_metadata_read=ignore_metadata_read)
     return cubes
 
 
-def main(source_path, output_path, grid_staggering, netcdf_only):
+def main(
+    source_path,
+    output_path,
+    grid_staggering,
+    netcdf_only,
+    ignore_metadata_read,
+    ignore_metadata_write,
+):
     """
     Convert specified source file to an ancillary.
 
@@ -58,6 +65,12 @@ def main(source_path, output_path, grid_staggering, netcdf_only):
         Output ancillary file path.  A NetCDF will also be written.
     grid_staggering : :obj:`int`, optional
         Grid staggering lookup, see UM F03 - FLH(9).
+    netcdf_only : bool
+        If True, only write to netCDF. If False, write to both netCDF and ancillary.
+    ignore_metadata_read : :obj:`bool`, optional
+        If True, do not read metadata sidecar files alongside the source.
+    ignore_metadata_write : :obj:`bool`, optional
+        If True, do not write metadata sidecar files alongside the ancillary.
 
     Returns
     -------
@@ -66,13 +79,17 @@ def main(source_path, output_path, grid_staggering, netcdf_only):
         written to an ancillary.
 
     """
-    source_cubes = load_data(source_path)
+    source_cubes = load_data(source_path, ignore_metadata_read)
     if grid_staggering is not None:
         for source_cube in source_cubes:
             source_cube.attributes["grid_staggering"] = grid_staggering
 
     if not netcdf_only:
-        save.ancil(source_cubes, output_path)
+        save.ancil(
+            source_cubes,
+            output_path,
+            ignore_metadata_write=ignore_metadata_write,
+        )
     save.netcdf(source_cubes, output_path)
 
     return source_cubes
@@ -98,6 +115,8 @@ def cli_interface():
         args.output,
         args.grid_staggering,
         args.netcdf_only,
+        args.ignore_metadata_read,
+        args.ignore_metadata_write,
     )
 
 

lib/ants/cli/ancil_fill_n_merge.py

@@ -25,6 +25,7 @@ def load_data(
     land_fraction_threshold=None,
     begin=None,
     end=None,
+    ignore_metadata_read=False,
 ):
     """
     Load the necessary data for performing a merge and fill operation.
@@ -45,6 +46,9 @@ def load_data(
         Datetime to start the processing.
     end: :obj:`datetime`, optional
         Datetime to end the processing.
+    ignore_metadata_read : :obj:`bool`, optional
+        When set to True, files containing metadata will not be loaded alongside data
+        and added as attributes to the cube.
 
 
     Returns
@@ -56,12 +60,16 @@ def load_data(
         respectively.
 
     """
-    primary_cubes = ants.io.load.load(primary_source)
+    primary_cubes = ants.io.load.load(
+        primary_source, ignore_metadata_read=ignore_metadata_read
+    )
     if begin is not None:
         primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end)
     alternate_cubes = None
     if alternate_source:
-        alternate_cubes = ants.io.load.load(alternate_source)
+        alternate_cubes = ants.io.load.load(
+            alternate_source, ignore_metadata_read=ignore_metadata_read
+        )
         if begin is not None:
             alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end)
 
@@ -97,6 +105,8 @@ def main(
     netcdf_only,
     search_method,
     blending_distance,
+    ignore_metadata_read,
+    ignore_metadata_write,
 ):
     """
     Perform merge and fill operation on the provided sources.
@@ -152,6 +162,12 @@ def main(
         is applied. Note that this is in units of grid cells, not a physical distance.
         If ``None``, no blending is applied, and there will be a hard edge between
         the two sources.
+    ignore_metadata_read : :obj:`bool`, optional
+        When set to True, files containing metadata will not be loaded alongside data
+        and added as attributes to the cube.
+    ignore_metadata_write : bool
+        If True, do not write metadata sidecar files alongside the ancillary.
+
 
     Returns
     -------
@@ -171,6 +187,7 @@ def main(
         land_fraction_threshold,
         begin,
         end,
+        ignore_metadata_read,
     )
 
     result = primary_cubes
@@ -182,7 +199,7 @@ def main(
         ants.analysis.make_consistent_with_lsm(result, lbm, invert_mask, search_method)
 
     if not netcdf_only:
-        save.ancil(result, output)
+        save.ancil(result, output, ignore_metadata_write=ignore_metadata_write)
     save.netcdf(result, output)
 
     return result
@@ -271,6 +288,8 @@ def cli_interface():
         netcdf_only=args.netcdf_only,
         search_method=args.search_method,
         blending_distance=args.blending_distance,
+        ignore_metadata_read=args.ignore_metadata_read,
+        ignore_metadata_write=args.ignore_metadata_write,
     )
 
 

lib/ants/cli/ancil_general_regrid.py

@@ -42,15 +42,20 @@ def load_data(
     land_fraction_threshold=None,
     begin=None,
     end=None,
+    ignore_metadata_read=False,
 ):
-    source_cubes = ants.io.load.load(source)
+    source_cubes = ants.io.load.load(source, ignore_metadata_read=ignore_metadata_read)
     if begin is not None:
         source_cubes = create_time_constrained_cubes(source_cubes, begin, end)
     if target_grid:
-        target_cube = ants.io.load.load_grid(target_grid)
+        target_cube = ants.io.load.load_grid(
+            target_grid, ignore_metadata_read=ignore_metadata_read
+        )
     else:
         target_cube = ants.io.load.load_landsea_mask(
-            target_landseamask, land_fraction_threshold
+            target_landseamask,
+            land_fraction_threshold,
+            ignore_metadata_read=ignore_metadata_read,
         )
     return source_cubes, target_cube
 
@@ -76,6 +81,8 @@ def main(
     save_ukca,
     netcdf_only,
     search_method,
+    ignore_metadata_read,
+    ignore_metadata_write,
 ):
     """
     General regrid application top level call function.
@@ -118,6 +125,11 @@ def main(
         provided source(s) consistent with the provided land sea mask.
         This should only be provided if a target land sea mask is also
         provided via target_lsm_path.
+    ignore_metadata_read : :obj:`bool`, optional
+        When set to True, files containing metadata will not be loaded alongside data
+        and added as attributes to the cube.
+    ignore_metadata_write : :obj:`bool`, optional
+        If True, do not write metadata sidecar files alongside the ancillary.
 
     Returns
     -------
@@ -132,6 +144,7 @@ def main(
         land_fraction_threshold,
         begin,
         end,
+        ignore_metadata_read,
     )
     if ants.utils.cube._is_ugrid(target_cube):
         raise ValueError(
@@ -149,7 +162,11 @@ def main(
         save.ukca_netcdf(regridded_cubes, output_path)
     else:
         if not netcdf_only:
-            save.ancil(regridded_cubes, output_path)
+            save.ancil(
+                regridded_cubes,
+                output_path,
+                ignore_metadata_write=ignore_metadata_write,
+            )
         save.netcdf(regridded_cubes, output_path)
 
     return regridded_cubes
@@ -210,6 +227,8 @@ def cli_interface():
         args.save_ukca,
         args.netcdf_only,
         args.search_method,
+        args.ignore_metadata_read,
+        args.ignore_metadata_write,
     )
 
 

lib/ants/cli/ancil_vertical_regrid.py

@@ -25,8 +25,9 @@ def load_data(
     target_grid,
     begin=None,
     end=None,
+    ignore_metadata_read=None,
 ):
-    source_cubes = ants.io.load.load(source)
+    source_cubes = ants.io.load.load(source, ignore_metadata_read=ignore_metadata_read)
     if begin is not None:
         source_cubes = create_time_constrained_cubes(source_cubes, begin, end)
 
@@ -87,6 +88,8 @@ def main(
     end,
     save_ukca,
     netcdf_only,
+    ignore_metadata_read,
+    ignore_metadata_write,
 ):
     """
     Vertical regrid application top level call function.
@@ -112,6 +115,11 @@ def main(
     end : :obj:`datetime`, optional
         If provided, all source data after this year is discarded.  Default is to
         include all source data.
+    ignore_metadata_read : :obj:`bool`, optional
+        When set to True, files containing metadata will not be loaded alongside data
+        and added as attributes to the cube.
+    ignore_metadata_write : bool
+        If True, do not write metadata sidecar files alongside the ancillary.
 
 
     Returns
@@ -121,10 +129,7 @@ def main(
 
     """
     source_cubes, target_cube = load_data(
-        source_path,
-        target_path,
-        begin,
-        end,
+        source_path, target_path, begin, end, ignore_metadata_read
     )
 
     regridded_cubes = regrid(source_cubes, target_cube)
@@ -133,7 +138,11 @@ def main(
         save.ukca_netcdf(regridded_cubes, output_path)
     else:
         if not netcdf_only:
-            save.ancil(regridded_cubes, output_path)
+            save.ancil(
+                regridded_cubes,
+                output_path,
+                ignore_metadata_write=ignore_metadata_write,
+            )
         save.netcdf(regridded_cubes, output_path)
 
     return regridded_cubes
@@ -163,6 +172,8 @@ def cli_interface():
         args.end,
         args.save_ukca,
         args.netcdf_only,
+        args.ignore_metadata_read,
+        args.ignore_metadata_write,
     )