diff --git a/mkdocs/site/packages/evo-compute/ComputeClient.html b/mkdocs/site/packages/evo-compute/ComputeClient.html index c3ec360dc..b85ce073a 100644 --- a/mkdocs/site/packages/evo-compute/ComputeClient.html +++ b/mkdocs/site/packages/evo-compute/ComputeClient.html @@ -147,12 +147,11 @@

Check the parameters against the task's JSON Schema before submitting -(required-field presence, and, under deep_validation, the whole payload). +(required-field presence, and, under deep_validation, known value constraints). Defaults to True. False turns off deep validation too, whatever deep_validation says. It does not make the call unchecked: the parameters are -still bound to the signature synthesised from the schema, so an unknown or missing -required argument is still rejected. That is how the payload is built, not a -validation pass, and there is no useful call to make without it.

+still bound to the task's required arguments, and explicit extra-field rules +still apply. Custom runners also retain their own parameter checks.

@@ -168,7 +167,7 @@

-

Additionally run full JSON Schema Draft 2020-12 validation. +

Additionally check the full JSON Schema Draft 2020-12 constraints. Defaults to False. Only consulted when validate is True.

@@ -218,6 +217,30 @@

+
+ + + +

+ context + + + + property + + +

+
context: IContext
+
+ +
+ +

The context used to submit tasks and load their results.

+ +
+ +
+ @@ -241,183 +264,16 @@

validate: bool | None = None, deep_validation: bool | None = None, check_schemas: bool | None = None, -) -> TaskResult +) -> Any
-

Discover the task (cached), resolve and validate the parameters, submit, and hydrate the results.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
- topic - - str - -
-

The task topic.

-
-
- required -
- task - - str - -
-

The task name.

-
-
- required -
- parameters - - dict[str, Any] - -
-

The task parameters, plus preview=True to run a preview task.

-
-
- required -
- validate - - bool | None - -
-

Override the client's schema-validation setting for this call. -False skips deep validation too, but never the signature binding that -builds the payload.

-
-
- None -
- deep_validation - - bool | None - -
-

Override the client's deep-validation setting for this call. -Only consulted when validation is enabled.

-
-
- None -
- check_schemas - - bool | None - -
-

Override the client's supported-schema setting for this call. -Falls back to validate when neither is set.

-
-
- None -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- TaskResult - -
-

The hydrated results of the finished job.

-
-
- - -

Raises:

- - - - - - - - - - - - - - - - - -
TypeDescription
- AttributeError - -
-

If the topic does not advertise the task.

-
-
- ParameterValidationError - -
-

If the parameters or referenced objects fail validation.

-
-
+

Run a task by name, using its custom runner when one is registered.

+

Parameters use the same names as client.<topic>.<task>.run(...). +Custom tasks return their native result; other tasks return TaskResult. +Validation options override the client's settings for this call. +Preview is off unless parameters explicitly includes preview=True.

diff --git a/packages/evo-compute/README.md b/packages/evo-compute/README.md index d89fac0ad..da8eabdfd 100644 --- a/packages/evo-compute/README.md +++ b/packages/evo-compute/README.md @@ -38,6 +38,33 @@ To get up and running quickly with the Evo Compute Task SDK, start by configurin For some interactive Jupyter notebook examples, see the [examples folder](docs/examples). +### Running tasks through the generic engine + +Authenticate and load inputs through the existing async SDK APIs, then use a client on +the same event loop: + +```python +from evo.compute import ComputeClient + +client = ComputeClient(manager) +result = await client.geostatistics.kriging.run( + source=samples.attributes["grade"], + target=target_model.attributes["kriged_grade"], + variogram=variogram, + neighborhood=search, + preview=True, +) +dataframe = await result.to_dataframe() +``` + +Kriging opts into a handwritten runner with additional checks and native `KrigingResult`. +Its arguments use the same names as other estimation tasks: `neighborhood` and +`kriging_method`. Other tasks use the schema-driven generic surface. +`client.arun(topic, task, parameters)` uses the same runner, argument names and result type +as the task's `run(...)` method, including for tasks absent from the snapshot. +Preview is off by default on every path; pass `preview=True` to opt in. +The existing lowercase task modules and `await tasks.run(manager, parameters)` API are unchanged. + ### Static typing for the generic engine `ComputeClient` resolves `client...run(...)` from the live task catalogue, so @@ -46,6 +73,13 @@ editors completion, signature help and parameter checking for every task in a ch catalogue snapshot, while execution stays fully generic. See [stubs/README.md](stubs/README.md) for how to regenerate it and refresh the snapshot. +For compatible neighborhood schemas, generated client signatures accept the +existing `SearchNeighborhood` model alongside schema-shaped dictionaries. Its native +`Ellipsoid`, ranges and rotation objects keep their existing serialization; the type hints +describe behavior the resolver already supports. Matching is based on serialized field +structure, not a task or schema name. Unrelated models and malformed dictionaries are still +rejected by the type checkers. + ## Contributing For instructions on contributing to the development of this library, please refer to the [evo-python-sdk documentation](https://github.com/seequentevo/evo-python-sdk). diff --git a/packages/evo-compute/src/evo/compute/_stubgen.py b/packages/evo-compute/src/evo/compute/_stubgen.py index 238c3153e..756aa29ca 100644 --- a/packages/evo-compute/src/evo/compute/_stubgen.py +++ b/packages/evo-compute/src/evo/compute/_stubgen.py @@ -36,6 +36,7 @@ import argparse import ast +import inspect import json import keyword import re @@ -47,6 +48,7 @@ from .endpoints.models import TaskResource from .engine import _normalise, _wire_names +from .overrides import load_override, overridden_tasks __all__ = [ "capture_snapshot", @@ -480,7 +482,7 @@ def _run_parameters(self) -> list[str]: annotation = self._annotation(properties.get(name, {}), schema, (name,)) default = "" if name in required else " = ..." lines.append(f"{published[name]}: {annotation}{default}") - lines.append(f"preview: bool = {bool(self.spec.feature_flag)!r}") + lines.append("preview: bool = False") return lines def _return_type(self) -> str: @@ -640,7 +642,7 @@ def _render_client(topics: list[str]) -> list[str]: " validate: bool | None = ...,", " deep_validation: bool | None = ...,", " check_schemas: bool | None = ...,", - " ) -> TaskResult:", + " ) -> Any:", _docstring( "Run any task by name, including one this stub does not know about.", " ", @@ -654,8 +656,17 @@ def _render_client(topics: list[str]) -> list[str]: return lines +def _override_runner(topic: str, task: str) -> tuple[str, str]: + module = load_override(topic, task) + assert module is not None, "only called for a task an override claims" + annotation = inspect.signature(module.bind).return_annotation + return module.__name__.removeprefix("evo.compute"), getattr(annotation, "__name__", annotation) + + def _render(tasks: list[TaskResource], snapshot_dir: Path) -> str: + overridden = overridden_tasks() stubs: list[_TaskStub] = [] + override_imports: list[str] = [] members: dict[str, list[tuple[str, str]]] = {} def remember(topic: str, attribute: str, class_name: str) -> None: @@ -663,10 +674,18 @@ def remember(topic: str, attribute: str, class_name: str) -> None: for task in sorted(tasks, key=lambda task: (task.topic, task.name)): attribute = _normalise(task.name) + if (_normalise(task.topic), attribute) in overridden: + continue awaited = _TaskRenderer(task).render() stubs.append(awaited) remember(task.topic, attribute, awaited.class_name) + for topic, task_name in overridden: + class_name = f"_{_camel(topic)}{_camel(task_name)}" + module_path, runner_name = _override_runner(topic, task_name) + override_imports.append(f"from {module_path} import {runner_name} as {class_name}") + remember(topic, task_name, class_name) + blocks: list[str] = [] for stub in stubs: blocks.extend(generated.body for generated in stub.types) @@ -744,6 +763,7 @@ def remember(topic: str, attribute: str, class_name: str) -> None: if "AnyTypedAttribute" in declared else [] ), + *override_imports, ] ), ] diff --git a/packages/evo-compute/src/evo/compute/engine.py b/packages/evo-compute/src/evo/compute/engine.py index 5cb13dc64..457927cfb 100644 --- a/packages/evo-compute/src/evo/compute/engine.py +++ b/packages/evo-compute/src/evo/compute/engine.py @@ -31,6 +31,9 @@ The catalogue is fetched the first time a task is ``run(...)``. It is then held by the underlying :class:`~evo.compute.discovery.DiscoveryClient`, so repeated runs are served from there until its cache expires. + +A task can opt into a hand-written surface through :mod:`evo.compute.overrides`. +Named calls through ``arun`` use the same override as attribute-based calls. """ from __future__ import annotations @@ -47,6 +50,7 @@ from .endpoints.models import TaskResource from .exceptions import ParameterValidationError from .outputs import TaskResult +from .overrides import load_override from .resolution import ReferenceResolver from .validation import validate_parameters @@ -83,14 +87,12 @@ def _python_annotation(prop: dict[str, Any]) -> Any: def _wire_names(spec: TaskResource) -> dict[str, str]: - """Map each parameter's Python name to the name the task's schema uses. - - Empty when a parameter cannot have a keyword argument of its own: its normalised name - is not an identifier, is a reserved word, or collides with another parameter's. - """ + """Map unambiguous Python parameter names back to the catalogue's wire names.""" + override = load_override(_normalise(spec.topic), _normalise(spec.name)) + aliases = getattr(override, "PARAMETER_ALIASES", {}) if override is not None else {} names: dict[str, str] = {} for wire_name in (spec.parameters or {}).get("properties", {}): - python_name = _normalise(wire_name) + python_name = aliases.get(wire_name, _normalise(wire_name)) if not python_name.isidentifier() or keyword.iskeyword(python_name) or python_name in names: return {} names[python_name] = wire_name @@ -138,18 +140,12 @@ def _signature_from_schema(spec: TaskResource) -> inspect.Signature: return inspect.Signature(parameters, return_annotation=TaskResult) -def _check_fallback_parameters(spec: TaskResource, parameters: dict[str, Any], label: str) -> None: - """Enforce required and declared parameter names when ``bind`` uses ``**parameters``.""" - schema = spec.parameters or {} - properties: dict[str, Any] = schema.get("properties", {}) - for name in schema.get("required", []): +def _check_fallback_required(spec: TaskResource, parameters: dict[str, Any], label: str) -> None: + """Enforce required parameter names when ``bind`` uses ``**parameters``.""" + for name in (spec.parameters or {}).get("required", []): if name not in parameters: detail = f"missing a required keyword-only argument: {name!r}" raise ParameterValidationError(f"{label}.run(): {detail}", task=label, errors=[detail]) - for name in parameters: - if name not in properties: - detail = f"got an unexpected keyword argument {name!r}" - raise ParameterValidationError(f"{label}.run(): {detail}", task=label, errors=[detail]) class ComputeClient: @@ -161,13 +157,12 @@ class ComputeClient: :param context: An authenticated Evo context. :param cache_ttl_seconds: How long a discovered task catalogue is cached. :param validate: Check the parameters against the task's JSON Schema before submitting - (required-field presence, and, under ``deep_validation``, the whole payload). + (required-field presence, and, under ``deep_validation``, known value constraints). Defaults to ``True``. ``False`` turns off deep validation too, whatever ``deep_validation`` says. It does *not* make the call unchecked: the parameters are - still bound to the signature synthesised from the schema, so an unknown or missing - required argument is still rejected. That is how the payload is built, not a - validation pass, and there is no useful call to make without it. - :param deep_validation: Additionally run full JSON Schema Draft 2020-12 validation. + still bound to the task's required arguments, and explicit extra-field rules + still apply. Custom runners also retain their own parameter checks. + :param deep_validation: Additionally check the full JSON Schema Draft 2020-12 constraints. Defaults to ``False``. Only consulted when ``validate`` is ``True``. :param check_schemas: Check that each referenced geoscience object is of a schema the task declares support for. Defaults to following ``validate``. Independent of it @@ -195,6 +190,11 @@ def __init__( self._deep_validation = deep_validation self._check_schemas = check_schemas + @property + def context(self) -> IContext: + """The context used to submit tasks and load their results.""" + return self._context + # -- dynamic namespace ------------------------------------------------- # def __getattr__(self, name: str) -> _TopicProxy: @@ -231,12 +231,31 @@ async def arun( validate: bool | None = None, deep_validation: bool | None = None, check_schemas: bool | None = None, + ) -> Any: + """Run a task by name, using its custom runner when one is registered. + + Parameters use the same names as ``client...run(...)``. + Custom tasks return their native result; other tasks return ``TaskResult``. + Validation options override the client's settings for this call. + Preview is off unless ``parameters`` explicitly includes ``preview=True``. + """ + options = dict(validate=validate, deep_validation=deep_validation, check_schemas=check_schemas) + if (override := load_override(_normalise(topic), _normalise(task))) is not None: + return await override.bind(self, topic, task).run_parameters(parameters, **options) + return await self._run_generic(topic, task, parameters, **options) + + async def _run_generic( + self, + topic: str, + task: str, + parameters: dict[str, Any], + *, + validate: bool | None = None, + deep_validation: bool | None = None, + check_schemas: bool | None = None, ) -> TaskResult: """Discover the task (cached), resolve and validate the parameters, submit, and hydrate the results. - :param topic: The task topic. - :param task: The task name. - :param parameters: The task parameters, plus ``preview=True`` to run a preview task. :param validate: Override the client's schema-validation setting for this call. ``False`` skips deep validation too, but never the signature binding that builds the payload. @@ -244,17 +263,22 @@ async def arun( Only consulted when validation is enabled. :param check_schemas: Override the client's supported-schema setting for this call. Falls back to ``validate`` when neither is set. - - :return: The hydrated results of the finished job. - - :raises AttributeError: If the topic does not advertise the task. - :raises ParameterValidationError: If the parameters or referenced objects fail validation. """ spec = await self._resolve_spec(topic, task) label = f"{topic}.{_normalise(task)}" signature = _signature_from_schema(spec) + accepts_extra = any( + parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in signature.parameters.values() + ) + extra_parameters = ( + {} + if accepts_extra + else {name: value for name, value in parameters.items() if name not in signature.parameters} + ) try: - bound = signature.bind(**parameters) + bound = signature.bind( + **{name: value for name, value in parameters.items() if name not in extra_parameters} + ) except TypeError as error: raise ParameterValidationError(f"{label}.run(): {error}", task=label, errors=[str(error)]) from None @@ -268,7 +292,8 @@ async def arun( else: wire_parameters.update(wire_parameters.pop("parameters", {})) if (spec.parameters or {}).get("properties"): - _check_fallback_parameters(spec, wire_parameters, label) + _check_fallback_required(spec, wire_parameters, label) + wire_parameters.update(extra_parameters) if validate is None: validate = self._validate @@ -324,9 +349,11 @@ def __init__(self, client: ComputeClient, topic: str) -> None: self._client = client self._topic = topic - def __getattr__(self, name: str) -> _TaskProxy: + def __getattr__(self, name: str) -> Any: if name.startswith("_"): raise AttributeError(name) + if (override := load_override(_normalise(self._topic), _normalise(name))) is not None: + return override.bind(self._client, self._topic, name) return _TaskProxy(self._client, self._topic, name) def __dir__(self) -> list[str]: diff --git a/packages/evo-compute/src/evo/compute/engine.pyi b/packages/evo-compute/src/evo/compute/engine.pyi index ba3f46ce6..cdaeb87b8 100644 --- a/packages/evo-compute/src/evo/compute/engine.pyi +++ b/packages/evo-compute/src/evo/compute/engine.pyi @@ -27,6 +27,7 @@ from evo.objects.typed import BaseObject, DownloadedObject from typing_extensions import NotRequired, TypedDict from .outputs import ResultNode, TaskResult +from .overrides.geostatistics.kriging import KrigingOverride as _GeostatisticsKriging from .tasks.common.search import SearchNeighborhood from .tasks.common.source_target import AnyTypedAttribute @@ -2258,516 +2259,6 @@ class GeostatisticsKnnResult(TaskResult): target: GeostatisticsKnnResultTarget """The target that was created or updated.""" -class GeostatisticsKrigingSourceFilterCondition(TypedDict): - """ - A leaf filter condition applied to one attribute of the filtered object. - - Exactly one operator field must be provided. - - - ``in``: Membership test. Provide string category names for - ``CategoryAttribute`` objects or integers for integer-coded indices. - - ``not_in``: Exclusion test. - - ``equal``, ``not_equal``: Numeric equality/inequality. - - ``greater_than``, ``greater_than_or_equal_to``, ``less_than``, - ``less_than_or_equal_to``: Numeric comparisons on numeric attributes. - """ - - attribute: AttributeInput - """Attribute used to filter locations.""" - operator: Literal[ - "in", - "not_in", - "equal", - "not_equal", - "greater_than", - "greater_than_or_equal_to", - "less_than", - "less_than_or_equal_to", - ] - """The filter operator. Use 'in' or 'not_in' for category/membership tests (pair with 'values'). Use 'equal', 'not_equal', 'greater_than', 'greater_than_or_equal_to', 'less_than', or 'less_than_or_equal_to' for numeric comparisons (pair with 'threshold').""" - threshold: NotRequired[float | None] - """Numeric threshold for comparison. Required when operator is 'equal', 'not_equal', 'greater_than', 'greater_than_or_equal_to', 'less_than', or 'less_than_or_equal_to'.""" - type: NotRequired[Literal["condition"]] - """Identifies this as a leaf filter condition.""" - values: NotRequired[list[str] | None] - """Category values to match. Required when operator is 'in' or 'not_in'. Use strings for string/category-name attributes; use integers for integer-coded attributes.""" - -class GeostatisticsKrigingSourceAnyOfFilter(TypedDict): - """A composite filter that passes locations satisfying ANY child filter (OR logic).""" - - filters: list[ - GeostatisticsKrigingSourceFilterCondition - | GeostatisticsKrigingSourceAllOfFilter - | GeostatisticsKrigingSourceAnyOfFilter - ] - """At least one of these filters must be satisfied (OR logic).""" - type: NotRequired[Literal["any_of"]] - """Identifies this as an OR composite filter.""" - -class GeostatisticsKrigingSourceAllOfFilter(TypedDict): - """A composite filter that passes locations satisfying ALL child filters (AND logic).""" - - filters: list[ - GeostatisticsKrigingSourceFilterCondition - | GeostatisticsKrigingSourceAllOfFilter - | GeostatisticsKrigingSourceAnyOfFilter - ] - """All of these filters must be satisfied (AND logic).""" - type: NotRequired[Literal["all_of"]] - """Identifies this as an AND composite filter.""" - -class GeostatisticsKrigingSourceFilter(TypedDict): - """Optional filter to apply to the source object. This allows kriging to use only a subset of the known data, defined by filtering an attribute on the source object.""" - - where: ( - GeostatisticsKrigingSourceFilterCondition - | GeostatisticsKrigingSourceAllOfFilter - | GeostatisticsKrigingSourceAnyOfFilter - ) - """A filter expression defining which locations to include. Use a FilterCondition for a single predicate, AllOfFilter for AND logic, AnyOfFilter for OR logic, or nest them to build compound predicates.""" - -class GeostatisticsKrigingSource(TypedDict): - """The source object and attribute containing known values.""" - - attribute: AttributeInput - """The known values, used to estimate values at unknown locations.""" - filter: NotRequired[GeostatisticsKrigingSourceFilter | None] - """Optional filter to apply to the source object. This allows kriging to use only a subset of the known data, defined by filtering an attribute on the source object.""" - object: ObjectInput - """Spatial locations of known values.""" - -class GeostatisticsKrigingTargetAttributeCreate(TypedDict): - name: str - """The name of the attribute to create.""" - operation: Literal["create"] - """The operation to perform on the attribute.""" - -class GeostatisticsKrigingTargetAttributeUpdate(TypedDict): - operation: Literal["update"] - """The operation to perform on the attribute.""" - reference: AttributeInput - """Reference to an existing attribute to update.""" - -class GeostatisticsKrigingDiagnostics(TypedDict): - """Optional per-location diagnostic attributes to compute and write alongside the kriging result. Only diagnostics with an explicit attribute target are produced.""" - - aniso_mean_distance: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the mean anisotropic (ellipsoid-space) distance from the target location to all neighbours.""" - aniso_min_distance: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the minimum anisotropic (ellipsoid-space) distance from the target location to any neighbour.""" - kriging_efficiency: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the proportion of point or intra-block variance explained by the kriging weights.""" - kriging_mean: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the GLS mean for ordinary kriging, or the supplied constant mean for simple kriging.""" - kriging_variance: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the kriging variance (estimation uncertainty).""" - mean_distance: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the mean isotropic Euclidean distance from the target location to all neighbours.""" - min_distance: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the minimum isotropic Euclidean distance from the target location to any neighbour.""" - num_drillholes: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the number of distinct drillholes contributing to the estimate. Requires a DownholeIntervals source object.""" - num_duplicates: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the number of duplicate sample locations used in the estimate.""" - num_equidistant: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with a hint of how many other samples could have replaced the last returned sample because they were the same distance from the target location. This is counted after searching and before clipping to the maximum number of samples.""" - num_samples: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the number of data samples used in the estimate.""" - slope_of_regression: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the slope of regression, a diagnostic for conditional bias.""" - sum_negative_weights: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the sum of negative kriging weights applied to data samples (zero when negative-weight removal is enabled).""" - sum_positive_weights: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the sum of positive kriging weights applied to data samples.""" - sum_weights: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with the sum of kriging weights applied to data samples.""" - valid: NotRequired[ - GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - ] - """Attribute to create or update with whether each target location's neighbourhood satisfied the search constraints and produced an estimate.""" - -class GeostatisticsKrigingAnyOfFilter(TypedDict): - """A composite filter that passes locations satisfying ANY child filter (OR logic).""" - - filters: list[ - GeostatisticsKrigingSourceFilterCondition | GeostatisticsKrigingAllOfFilter | GeostatisticsKrigingAnyOfFilter - ] - """At least one of these filters must be satisfied (OR logic).""" - type: NotRequired[Literal["any_of"]] - """Identifies this as an OR composite filter.""" - -class GeostatisticsKrigingAllOfFilter(TypedDict): - """A composite filter that passes locations satisfying ALL child filters (AND logic).""" - - filters: list[ - GeostatisticsKrigingSourceFilterCondition | GeostatisticsKrigingAllOfFilter | GeostatisticsKrigingAnyOfFilter - ] - """All of these filters must be satisfied (AND logic).""" - type: NotRequired[Literal["all_of"]] - """Identifies this as an AND composite filter.""" - -class GeostatisticsKrigingFilter(TypedDict): - """Optional filter to apply to the target object. This allows the kriging to only be applied to a subset of the target object, defined by filtering an attribute on the target object.""" - - where: GeostatisticsKrigingSourceFilterCondition | GeostatisticsKrigingAllOfFilter | GeostatisticsKrigingAnyOfFilter - """A filter expression defining which locations to include. Use a FilterCondition for a single predicate, AllOfFilter for AND logic, AnyOfFilter for OR logic, or nest them to build compound predicates.""" - -class GeostatisticsKrigingTarget(TypedDict): - """The target object and attribute to create or update with kriging results.""" - - attribute: GeostatisticsKrigingTargetAttributeCreate | GeostatisticsKrigingTargetAttributeUpdate | AttributeInput - """Attribute to create or update on the target object with the kriging estimate. Use `{"operation": "create", "name": ...}` to add a new attribute, or `{"operation": "update", "reference": ...}` to overwrite an existing one.""" - diagnostics: NotRequired[GeostatisticsKrigingDiagnostics | None] - """Optional per-location diagnostic attributes to compute and write alongside the kriging result. Only diagnostics with an explicit attribute target are produced.""" - filter: NotRequired[GeostatisticsKrigingFilter | None] - """Optional filter to apply to the target object. This allows the kriging to only be applied to a subset of the target object, defined by filtering an attribute on the target object.""" - object: ObjectInput - """Object to evaluate onto.""" - update_mode: NotRequired[Literal["auto", "merge", "replace"]] - """How to treat target locations that this run did not estimate, such as locations excluded by the target filter or where the neighbourhood search found no samples. 'merge' keeps the values those locations already had, so running one domain at a time accumulates results into a single attribute. 'replace' clears them, so the attribute only ever holds the results of the latest run. 'auto' (the default) merges when a target filter is set and replaces otherwise. Merging only applies to attributes that already exist on the target object; a newly created attribute has nothing to merge with. Boolean and integer diagnostic attributes have no way to represent 'not estimated', so within the filter they are always written as produced.""" - -class GeostatisticsKrigingSimpleKriging(TypedDict): - """Kriging method with a constant mean value.""" - - mean: float - """The mean value, assumed to be constant across the domain.""" - type: Literal["simple"] - """The type of kriging to use.""" - -class GeostatisticsKrigingOrdinaryKriging(TypedDict): - """Kriging method with a variable mean value.""" - - type: Literal["ordinary"] - """The type of kriging to use.""" - -class GeostatisticsKrigingNeighborhoodEllipsoidEllipsoidRanges(TypedDict): - """The ranges of the ellipsoid""" - - major: float - """The major axis length of the ellipsoid""" - minor: float - """The minor axis length of the ellipsoid""" - semi_major: float - """The semi major axis length of the ellipsoid""" - -class GeostatisticsKrigingRotation(TypedDict): - """The rotation of the ellipsoid""" - - dip: NotRequired[float] - """second rotation, about the x-axis, in degrees""" - dip_azimuth: NotRequired[float] - """first rotation, about the z-axis, in degrees""" - pitch: NotRequired[float] - """third rotation, about the z-axis, in degrees""" - -class GeostatisticsKrigingEllipsoid(TypedDict): - """ - The ellipsoid, to search for points within - Alias: anisotropy - """ - - ellipsoid_ranges: GeostatisticsKrigingNeighborhoodEllipsoidEllipsoidRanges - """The ranges of the ellipsoid""" - rotation: GeostatisticsKrigingRotation - """The rotation of the ellipsoid""" - -class GeostatisticsKrigingFractionalDistance(TypedDict): - """Outlier distance specified as a fraction of the search radius.""" - - distance: float - """The distance to the outlier limit, as a fraction of the search radius (0.0 to 1.0). For example, a value of 0.5 means the outlier boundary is at 50% of the search radius.""" - type: Literal["fraction"] - """Specify the distance as a fraction of the search radius.""" - -class GeostatisticsKrigingAbsoluteDistance(TypedDict): - """Outlier distance specified as an absolute value.""" - - distance: float - """The absolute distance to the outlier limit. This cannot be used yet.""" - type: Literal["absolute"] - """Specify the distance as an absolute value.""" - -class GeostatisticsKrigingOutlierRestriction(TypedDict): - """Outlier restrictions.""" - - action: Literal["clamp", "discard"] - """The action to take on outliers. Clamp the value, or discard the point.""" - clamp_delta: NotRequired[float] - """The maximum randomly-applied delta value for clamped outlier values.Random delta values in the range [0.0...clamp_limit) are added to clamped values toavoid identical values occurring throughout the samples.""" - distance: GeostatisticsKrigingFractionalDistance | GeostatisticsKrigingAbsoluteDistance - """The distance to the outlier limit.""" - max_value_threshold: NotRequired[float] - """The maximum allowed value for the outlier limit. Values exceeding this threshold are either discarded or clamped.""" - -class GeostatisticsKrigingNeighborhoodWithOutlierRestrictions(TypedDict): - """In Kriging, the value of each evaluation point is determined by a set of nearby points with known values. The search parameters determines which nearby points to use.""" - - ellipsoid: GeostatisticsKrigingEllipsoid - """ - The ellipsoid, to search for points within - Alias: anisotropy - """ - max_drillholes_per_estimate: NotRequired[int | None] - """The maximum number of drillholes to be used in each estimate.""" - max_empty_octants: NotRequired[int] - """The maximum number of empty octants (aka sectors) allowed when searching for neighbors. Omitting this, or a value of 8, disables the octant check.""" - max_empty_quadrants: NotRequired[int | None] - """The maximum number of empty quadrants (2D sectors, ignoring Z) allowed when searching for neighbors. If not specified, quadrant filtering is disabled.""" - max_samples: int - """ - The maximum number of samples to use for each evaluation point. - Alias: max-samples - """ - max_samples_per_drillhole: NotRequired[int | None] - """The maximum number of samples to use from each drillhole (defined by drillhole ID)""" - max_samples_per_octant: NotRequired[int | None] - """The maximum number of samples to use from each octant.""" - max_samples_per_quadrant: NotRequired[int | None] - """The maximum number of samples to use from each quadrant (2D sectors, ignoring Z).""" - min_samples: NotRequired[int] - """ - The minimum number of samples to use for each evaluation point. - Alias: min-samples - """ - outliers: NotRequired[GeostatisticsKrigingOutlierRestriction | None] - """Outlier restrictions.""" - -class GeostatisticsKrigingBlockDiscretization(TypedDict): - """Sub-block discretization for block kriging. When provided, each target block is subdivided into nx * ny * nz sub-cells and the kriged value is averaged across these sub-cells. When omitted, point kriging is performed. Only applicable when the target is a 3D grid or block model.""" - - nx: NotRequired[int] - """Number of subdivisions in the x direction""" - ny: NotRequired[int] - """Number of subdivisions in the y direction""" - nz: NotRequired[int] - """Number of subdivisions in the z direction""" - -class GeostatisticsKrigingResultAttribute(ResultNode): - """Attribute containing the kriging result.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsAnisoMeanDistance(ResultNode): - """Attribute containing the mean anisotropic (ellipsoid-space) distance from the target location to all neighbours.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsAnisoMinDistance(ResultNode): - """Attribute containing the minimum anisotropic (ellipsoid-space) distance from the target location to any neighbour.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsKrigingEfficiency(ResultNode): - """Attribute containing the proportion of point or intra-block variance explained by kriging weights.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsKrigingMean(ResultNode): - """Attribute containing the GLS mean for ordinary kriging, or the supplied constant mean for simple kriging.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsKrigingVariance(ResultNode): - """Attribute containing the kriging variance (estimation uncertainty).""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsMeanDistance(ResultNode): - """Attribute containing the mean isotropic Euclidean distance from the target location to all neighbours.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsMinDistance(ResultNode): - """Attribute containing the minimum isotropic Euclidean distance from the target location to any neighbour.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsNumDrillholes(ResultNode): - """Attribute containing the number of distinct drillholes contributing to the estimate.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsNumDuplicates(ResultNode): - """Attribute containing the number of duplicate sample locations used in the estimate.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsNumEquidistant(ResultNode): - """Attribute containing a hint of how many other samples could have replaced the last returned sample because they were the same distance from the target location. This is counted after searching and before clipping to the maximum number of samples.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsNumSamples(ResultNode): - """Attribute containing the number of data samples used in the estimate.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsSlopeOfRegression(ResultNode): - """Attribute containing the diagnostic for conditional bias.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsSumNegativeWeights(ResultNode): - """Attribute containing the sum of negative kriging weights applied to data samples (zero when negative-weight removal is enabled).""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsSumPositiveWeights(ResultNode): - """Attribute containing the sum of positive kriging weights applied to data samples.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultResultTargetDiagnosticsSumWeights(ResultNode): - """Attribute containing the sum of kriging weights applied to data samples.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultValid(ResultNode): - """Attribute containing whether each target location's neighbourhood satisfied the search constraints and produced an estimate.""" - - name: str - """The name of the output attribute.""" - reference: str - """Reference to the attribute in the geoscience object.""" - -class GeostatisticsKrigingResultDiagnostics(ResultNode): - """Diagnostic attributes produced alongside the kriging result, when requested.""" - - aniso_mean_distance: GeostatisticsKrigingResultResultTargetDiagnosticsAnisoMeanDistance | None - """Attribute containing the mean anisotropic (ellipsoid-space) distance from the target location to all neighbours.""" - aniso_min_distance: GeostatisticsKrigingResultResultTargetDiagnosticsAnisoMinDistance | None - """Attribute containing the minimum anisotropic (ellipsoid-space) distance from the target location to any neighbour.""" - kriging_efficiency: GeostatisticsKrigingResultResultTargetDiagnosticsKrigingEfficiency | None - """Attribute containing the proportion of point or intra-block variance explained by kriging weights.""" - kriging_mean: GeostatisticsKrigingResultResultTargetDiagnosticsKrigingMean | None - """Attribute containing the GLS mean for ordinary kriging, or the supplied constant mean for simple kriging.""" - kriging_variance: GeostatisticsKrigingResultResultTargetDiagnosticsKrigingVariance | None - """Attribute containing the kriging variance (estimation uncertainty).""" - mean_distance: GeostatisticsKrigingResultResultTargetDiagnosticsMeanDistance | None - """Attribute containing the mean isotropic Euclidean distance from the target location to all neighbours.""" - min_distance: GeostatisticsKrigingResultResultTargetDiagnosticsMinDistance | None - """Attribute containing the minimum isotropic Euclidean distance from the target location to any neighbour.""" - num_drillholes: GeostatisticsKrigingResultResultTargetDiagnosticsNumDrillholes | None - """Attribute containing the number of distinct drillholes contributing to the estimate.""" - num_duplicates: GeostatisticsKrigingResultResultTargetDiagnosticsNumDuplicates | None - """Attribute containing the number of duplicate sample locations used in the estimate.""" - num_equidistant: GeostatisticsKrigingResultResultTargetDiagnosticsNumEquidistant | None - """Attribute containing a hint of how many other samples could have replaced the last returned sample because they were the same distance from the target location. This is counted after searching and before clipping to the maximum number of samples.""" - num_samples: GeostatisticsKrigingResultResultTargetDiagnosticsNumSamples | None - """Attribute containing the number of data samples used in the estimate.""" - slope_of_regression: GeostatisticsKrigingResultResultTargetDiagnosticsSlopeOfRegression | None - """Attribute containing the diagnostic for conditional bias.""" - sum_negative_weights: GeostatisticsKrigingResultResultTargetDiagnosticsSumNegativeWeights | None - """Attribute containing the sum of negative kriging weights applied to data samples (zero when negative-weight removal is enabled).""" - sum_positive_weights: GeostatisticsKrigingResultResultTargetDiagnosticsSumPositiveWeights | None - """Attribute containing the sum of positive kriging weights applied to data samples.""" - sum_weights: GeostatisticsKrigingResultResultTargetDiagnosticsSumWeights | None - """Attribute containing the sum of kriging weights applied to data samples.""" - valid: GeostatisticsKrigingResultValid | None - """Attribute containing whether each target location's neighbourhood satisfied the search constraints and produced an estimate.""" - -class GeostatisticsKrigingResultTarget(ResultNode): - """The target that was created or updated.""" - - attribute: GeostatisticsKrigingResultAttribute - """Attribute containing the kriging result.""" - description: str | None - """The description of the geoscience object.""" - diagnostics: GeostatisticsKrigingResultDiagnostics | None - """Diagnostic attributes produced alongside the kriging result, when requested.""" - name: str - """The name of the geoscience object.""" - reference: str - """Reference to a geoscience object.""" - schema_id: str - """The ID of the Geoscience Object schema.""" - -class GeostatisticsKrigingResult(TaskResult): - """Result of the kriging task.""" - - message: str - """A message that says what happened in the task.""" - target: GeostatisticsKrigingResultTarget - """The target that was created or updated.""" - class GeostatisticsLocationWiseSource(TypedDict): """Source object and attribute containing ensemble values.""" @@ -5093,7 +4584,7 @@ class _ConverterDetectConverter: self, *, files: list[FileInput], - preview: bool = True, + preview: bool = False, ) -> ConverterDetectConverterResult: """Accepts one or more file references, inspects each file's extension, and returns which converter task(s) and geoscience object schema(s) are available for importing each file.""" ... @@ -5109,7 +4600,7 @@ class _ConverterDgnImport: epsg_code: int | None = ..., include_fills: bool = ..., include_text_anchors: bool = ..., - preview: bool = True, + preview: bool = False, ) -> ConverterDgnImportResult: """Import DGN files into Evo as line segments, with fallback to triangle mesh or pointset.""" ... @@ -5123,7 +4614,7 @@ class _ConverterErsImport: file: FileInput, file_data: FileInput, target: ConverterErsImportTarget, - preview: bool = True, + preview: bool = False, ) -> ConverterErsImportResult: """Import ER Mapper Raster (.ers) files into Evo as Regular-2D geoscience objects.""" ... @@ -5137,7 +4628,7 @@ class _ConverterGeosoftGridImport: file: FileInput, target: ConverterGeosoftGridImportTarget, file_gi: FileInput | None = ..., - preview: bool = True, + preview: bool = False, ) -> ConverterGeosoftGridImportResult: """Import Geosoft Grid (.grd/.grd.gi) files into Evo as Regular-2D geoscience objects.""" ... @@ -5150,7 +4641,7 @@ class _ConverterGeosoftSurfaceImport: *, file: FileInput, target: ConverterGeosoftSurfaceImportTarget, - preview: bool = True, + preview: bool = False, ) -> ConverterGeosoftSurfaceImportResult: """Import Geosoft Surface (.geosoft_surface) files into Evo as GeologicalModelMeshes geoscience objects.""" ... @@ -5163,7 +4654,7 @@ class _ConverterGeosoftVoxelImport: *, file: FileInput, target: ConverterGeosoftVoxelImportTarget, - preview: bool = True, + preview: bool = False, ) -> ConverterGeosoftVoxelImportResult: """Import Geosoft Voxel (.geosoft_voxel) files into Evo as either Regular-3D or Tensor-3D geoscience objects.""" ... @@ -5179,7 +4670,7 @@ class _ConverterImageImport: cell_size: list[float] | None = ..., coordinate_reference_system: str = ..., origin: list[float] | None = ..., - preview: bool = True, + preview: bool = False, ) -> ConverterImageImportResult: """Import image files into Evo as Regular-2D geoscience objects.""" ... @@ -5193,7 +4684,7 @@ class _ConverterObjImport: file: FileInput, target: ConverterObjImportTarget, coordinate_reference_system: str | None = ..., - preview: bool = True, + preview: bool = False, ) -> ConverterObjImportResult: """Import OBJ (Wavefront) files into Evo as Triangle Mesh geoscience objects. Supports triangles, quads (auto-triangulated), and multiple objects within a single file.""" ... @@ -5213,7 +4704,7 @@ class _ConverterShpImport: file_sbn: FileInput | None = ..., file_sbx: FileInput | None = ..., file_xml: FileInput | None = ..., - preview: bool = True, + preview: bool = False, ) -> ConverterShpImportResult: """Task to import SHP files into Evo as Triangle Mesh geoscience objects.""" ... @@ -5228,7 +4719,7 @@ class _ConverterUbcImport: file_mod: FileInput, target: ConverterUbcImportTarget, coordinate_reference_system: str = ..., - preview: bool = True, + preview: bool = False, ) -> ConverterUbcImportResult: """Task to import UBC Grids into Evo as Tensor-3D geoscience objects.""" ... @@ -5240,7 +4731,7 @@ class _ConverterVtkImport: self, *, files: list[FileInput], - preview: bool = True, + preview: bool = False, ) -> ConverterVtkImportResult: """VTK Import compute task""" ... @@ -5257,7 +4748,7 @@ class _ConverterXyzImport: x_index: int | None = ..., y_index: int | None = ..., z_index: int | None = ..., - preview: bool = True, + preview: bool = False, ) -> ConverterXyzImportResult: """Task to import XYZ files into Evo as Pointset geoscience objects.""" ... @@ -5270,7 +4761,7 @@ class _CptToBoreholeCptReader: *, cpt_file: FileInput, target_downhole_collection: CptToBoreholeCptReaderTargetDownholeCollection, - preview: bool = True, + preview: bool = False, ) -> CptToBoreholeCptReaderResult: """CPT reader task""" ... @@ -5286,7 +4777,7 @@ class _CptToBoreholeCreateClusters: hole_ids: list[str] | None = ..., number_of_clusters: int | None = ..., uncluster: bool = ..., - preview: bool = True, + preview: bool = False, ) -> CptToBoreholeCreateClustersResult: """Create clusters of related soil regions based on the stratification performed by the stratify task.""" ... @@ -5302,7 +4793,7 @@ class _CptToBoreholePublish: include_clusters: bool, cluster_metadata: list[CptToBoreholePublishClusterMetadata] | None = ..., exclude_collections: list[Literal["measured_cpt_readings", "derived_cpt_readings", "geology_layers"]] = ..., - preview: bool = True, + preview: bool = False, ) -> CptToBoreholePublishResult: """Publish a finalized downhole collection as a new geoscience object for downstream consumption.""" ... @@ -5317,7 +4808,7 @@ class _CptToBoreholeStratify: target_downhole_collection: CptToBoreholeStratifyTargetDownholeCollection, cpt_parameters: list[CptToBoreholeStratifyPerHoleParameters] | None = ..., execution_mode: Literal["inline", "fan_out", "auto"] = ..., - preview: bool = True, + preview: bool = False, ) -> CptToBoreholeStratifyResult: """Classifies and stratifies CPT data into geological layers based on predefined criteria and algorithms, facilitating geotechnical analysis and modeling.""" ... @@ -5330,7 +4821,7 @@ class _DriverDriverGeneralCompute: *, worker_message_body_json: str, _backend_url: str = ..., - preview: bool = True, + preview: bool = False, ) -> DriverDriverGeneralComputeResult: """Driver general compute task""" ... @@ -5346,7 +4837,7 @@ class _GeophysicsGravityTerrainCorrection: correction_radius: float, terrain_layer: GeophysicsGravityTerrainCorrectionTerrainLayer, water_layer: GeophysicsGravityTerrainCorrectionWaterLayer | None = ..., - preview: bool = True, + preview: bool = False, ) -> GeophysicsGravityTerrainCorrectionResult: """Gravity terrain correction compute task for geophysical surveys""" ... @@ -5360,7 +4851,7 @@ class _GeophysicsRegular2dGridToTrimesh: source: GeophysicsRegular2dGridToTrimeshSourceInput | AnyTypedAttribute, target: GeophysicsRegular2dGridToTrimeshTarget, settings: GeophysicsRegular2dGridToTrimeshSettings = ..., - preview: bool = True, + preview: bool = False, ) -> GeophysicsRegular2dGridToTrimeshResult: """Convert a regular 2D grid to a triangle mesh using a cell-centered or vertex-centered scalar attribute as Z values. Supports optional NaN filtering.""" ... @@ -5375,7 +4866,7 @@ class _GeostatisticsBreakTies: target: GeostatisticsBreakTiesTarget | AnyTypedAttribute, neighborhood: GeostatisticsBreakTiesBreakTiesSearch | SearchNeighborhood, seed: int = ..., - preview: bool = True, + preview: bool = False, ) -> GeostatisticsBreakTiesResult: """For more information, please read the guide""" ... @@ -5399,7 +4890,7 @@ class _GeostatisticsConditionalTurningBands: random_seed: int = ..., realizations: int = ..., source_filter: GeostatisticsConditionalTurningBandsSourceFilter | None = ..., - preview: bool = True, + preview: bool = False, ) -> GeostatisticsConditionalTurningBandsResult: """Runs the turning band simulator using an existing continuous distribution. For more information, please read the guide""" ... @@ -5446,7 +4937,7 @@ class _GeostatisticsContinuousDistribution: target: GeostatisticsContinuousDistributionTarget, tail_extrapolation: GeostatisticsContinuousDistributionTailExtrapolation | None = ..., weights: GeostatisticsContinuousDistributionWeights | None | AnyTypedAttribute = ..., - preview: bool = True, + preview: bool = False, ) -> GeostatisticsContinuousDistributionResult: """For more information, please read the guide""" ... @@ -5462,7 +4953,7 @@ class _GeostatisticsDeclustering: target: GeostatisticsDeclusteringTarget | AnyTypedAttribute, neighborhood: GeostatisticsDeclusteringExtendedNeighborhood | SearchNeighborhood, power: float | None = ..., - preview: bool = True, + preview: bool = False, ) -> GeostatisticsDeclusteringResult: """Computes grid-based declustering weights by measuring each sample's influence on evaluation locations. Supports both KNN (arithmetic mean) and IDW (inverse-distance weighted) modes via an optional power parameter.""" ... @@ -5480,7 +4971,7 @@ class _GeostatisticsExperimentalVariogram: | GeostatisticsExperimentalVariogramDownholeVariogram ], target: GeostatisticsExperimentalVariogramTarget, - preview: bool = True, + preview: bool = False, ) -> GeostatisticsExperimentalVariogramResult: """Computes one or more experimental (sample) variograms from a points or downhole intervals source object and writes them to a new experimental variogram object. A single call can mix omnidirectional, directional (azimuth/dip tolerance cone with optional bandwidth/bandheight) and downhole variograms, each with its own lag definition.""" ... @@ -5495,7 +4986,7 @@ class _GeostatisticsIdw: target: GeostatisticsIdwTarget | AnyTypedAttribute, neighborhood: GeostatisticsIdwExtendedNeighborhood | SearchNeighborhood, power: float, - preview: bool = True, + preview: bool = False, ) -> GeostatisticsIdwResult: """Inverse distance weighting estimation using weighted average of nearby sample values, where weights are inversely proportional to distance raised to a specified power.""" ... @@ -5509,28 +5000,11 @@ class _GeostatisticsKnn: source: GeostatisticsKnnSource | AnyTypedAttribute, target: GeostatisticsKnnTarget | AnyTypedAttribute, neighborhood: GeostatisticsKnnExtendedNeighborhood | SearchNeighborhood, - preview: bool = True, + preview: bool = False, ) -> GeostatisticsKnnResult: """Nearest neighbor estimation using arithmetic mean of nearby sample values.""" ... -class _GeostatisticsKriging: - """For more information, please read the guide""" - - async def run( - self, - *, - source: GeostatisticsKrigingSource | AnyTypedAttribute, - target: GeostatisticsKrigingTarget | AnyTypedAttribute, - kriging_method: GeostatisticsKrigingSimpleKriging | GeostatisticsKrigingOrdinaryKriging, - variogram: ObjectInput, - neighborhood: GeostatisticsKrigingNeighborhoodWithOutlierRestrictions | SearchNeighborhood, - block_discretisation: GeostatisticsKrigingBlockDiscretization | None = ..., - preview: bool = True, - ) -> GeostatisticsKrigingResult: - """For more information, please read the guide""" - ... - class _GeostatisticsLocationWise: """For more information, please read the guide""" @@ -5543,7 +5017,7 @@ class _GeostatisticsLocationWise: probability_above_cutoff: GeostatisticsLocationWiseProbabilityAboveCutoff | None = ..., quantiles: list[float] | None = ..., summary: bool | None = ..., - preview: bool = True, + preview: bool = False, ) -> GeostatisticsLocationWiseResult: """For more information, please read the guide""" ... @@ -5561,7 +5035,7 @@ class _GeostatisticsLossCalculation: mining_waste_cost: float, mining_ore_cost: float, metal_recovery_fraction: float, - preview: bool = True, + preview: bool = False, ) -> GeostatisticsLossCalculationResult: """Calculate loss given block model and economic parameters""" ... @@ -5576,7 +5050,7 @@ class _GeostatisticsNormalScore: source: GeostatisticsNormalScoreSource | AnyTypedAttribute, distribution: ObjectInput, target: GeostatisticsNormalScoreTarget | AnyTypedAttribute, - preview: bool = True, + preview: bool = False, ) -> GeostatisticsNormalScoreResult: """For more information, please read the guide""" ... @@ -5594,7 +5068,7 @@ class _GeostatisticsProfitCalculation: mining_waste_cost: float, mining_ore_cost: float, metal_recovery_fraction: float, - preview: bool = True, + preview: bool = False, ) -> GeostatisticsProfitCalculationResult: """Calculate profit given block model and economic parameters""" ... @@ -5623,7 +5097,7 @@ class _GeostatisticsSimulationReport: random_seed: int = ..., report_context: GeostatisticsSimulationReportReportContext | None = ..., report_mean_thresholds: GeostatisticsSimulationReportReportMeanThresholds | None = ..., - preview: bool = True, + preview: bool = False, ) -> GeostatisticsSimulationReportResult: """For more information, please read the guide""" ... @@ -5641,7 +5115,7 @@ class _GeotechAddToIntersectedGeometry: max_point: GeotechAddToIntersectedGeometryPoint3D = ..., min_point: GeotechAddToIntersectedGeometryPoint3D2 = ..., rotation_angle: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechAddToIntersectedGeometryResult: """Adds one or more triangle mesh objects to an existing intersected Geological Model Meshes object.""" ... @@ -5654,7 +5128,7 @@ class _GeotechCalculateMeshQualityHistogram: *, source: ObjectInput, bin_width: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechCalculateMeshQualityHistogramResult: """Calculates a normalized inverse condition number histogram for triangles in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5666,7 +5140,7 @@ class _GeotechCheckTriangleOrientation: self, *, source: ObjectInput, - preview: bool = True, + preview: bool = False, ) -> GeotechCheckTriangleOrientationResult: """Identifies triangles with inconsistent normal orientation relative to their neighbors in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5680,7 +5154,7 @@ class _GeotechCopyGeologicalModelMeshes: source: ObjectInput, target: GeotechCopyGeologicalModelMeshesTarget, remove_external_volumes: bool = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechCopyGeologicalModelMeshesResult: """Copies an existing set of geological model meshes, optionally removing external volumes from the result.""" ... @@ -5695,7 +5169,7 @@ class _GeotechCreateCrossSection: target: GeotechCreateCrossSectionTarget, section_origin: GeotechCreateCrossSectionPoint3D, section_dip_azimuth_degrees: float, - preview: bool = True, + preview: bool = False, ) -> GeotechCreateCrossSectionResult: """Intersects parametric volumes in a 3D Design Geometry with a vertical plane defined by a section origin and dip azimuth, producing a 2D Design Geometry with polyline boundaries.""" ... @@ -5708,7 +5182,7 @@ class _GeotechCreateGeologicalModelMeshes: *, source: ObjectInput, target: GeotechCreateGeologicalModelMeshesTarget, - preview: bool = True, + preview: bool = False, ) -> GeotechCreateGeologicalModelMeshesResult: """Decomposes an intersected mesh or existing GMM into per-volume geological model meshes, optionally removing the top (external) volume.""" ... @@ -5722,7 +5196,7 @@ class _GeotechDetectDegenerateTriangles: source: ObjectInput, cap_min_angle_degrees: float = ..., needle_threshold_ratio: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechDetectDegenerateTrianglesResult: """Identifies needle and cap triangles in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5734,7 +5208,7 @@ class _GeotechDetectDuplicatePoints: self, *, source: ObjectInput, - preview: bool = True, + preview: bool = False, ) -> GeotechDetectDuplicatePointsResult: """Identifies exact duplicate vertices in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5746,7 +5220,7 @@ class _GeotechDetectDuplicateTriangles: self, *, source: ObjectInput, - preview: bool = True, + preview: bool = False, ) -> GeotechDetectDuplicateTrianglesResult: """Identifies duplicate triangles (same vertex indices regardless of order) in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5759,7 +5233,7 @@ class _GeotechDetectFins: *, source: ObjectInput, max_area_ratio: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechDetectFinsResult: """Identifies fin-like connected components without closed boundaries in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5772,7 +5246,7 @@ class _GeotechDetectHoles: *, source: ObjectInput, hole_size_ratio_tolerance: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechDetectHolesResult: """Identifies holes in a Triangle Mesh or Geological Model Meshes object based on boundary edge analysis.""" ... @@ -5784,7 +5258,7 @@ class _GeotechDetectNonManifoldEdges: self, *, source: ObjectInput, - preview: bool = True, + preview: bool = False, ) -> GeotechDetectNonManifoldEdgesResult: """Identifies edges shared by more than two triangles in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5796,7 +5270,7 @@ class _GeotechDetectNonManifoldVertices: self, *, source: ObjectInput, - preview: bool = True, + preview: bool = False, ) -> GeotechDetectNonManifoldVerticesResult: """Identifies vertices with non-manifold topology in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5810,7 +5284,7 @@ class _GeotechDetectNonPartitioningSurfaces: source: ObjectInput, hole_size_ratio_tolerance: float = ..., max_area_ratio: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechDetectNonPartitioningSurfacesResult: """Identifies non-partitioning surfaces that are neither fins nor surfaces with holes in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5823,7 +5297,7 @@ class _GeotechDetectSelfIntersections: *, source: ObjectInput, tolerance: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechDetectSelfIntersectionsResult: """Identifies self-intersecting triangles in a Triangle Mesh or Geological Model Meshes object.""" ... @@ -5838,7 +5312,7 @@ class _GeotechFillHoles: target: GeotechFillHolesTarget, edges: list[list[int]], fill_mode_selection: Literal["Fill", "FillAndRefine", "FillRefineAndFair"], - preview: bool = True, + preview: bool = False, ) -> GeotechFillHolesResult: """Fills holes in the triangle mesh of a GMM object using specified edge identifiers and a configurable fill mode strategy.""" ... @@ -5852,7 +5326,7 @@ class _GeotechFixSelfIntersections: source: ObjectInput, target: GeotechFixSelfIntersectionsTarget, intersect_settings: GeotechFixSelfIntersectionsIntersectSettings, - preview: bool = True, + preview: bool = False, ) -> GeotechFixSelfIntersectionsResult: """Fixes self-intersections in the triangle mesh of a GMM object using configurable intersection detection settings.""" ... @@ -5868,7 +5342,7 @@ class _GeotechInitAggregateGeomGmm: max_point: GeotechInitAggregateGeomGmmPoint3D2, coordinate_reference_system: str | None = ..., rotation_angle: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechInitAggregateGeomGmmResult: """Creates a rotated bounding box as a Geological Model Meshes object, used as the initial intersected geometry when creating a model.""" ... @@ -5884,7 +5358,7 @@ class _GeotechLocalRemesh: seed_triangle_indices: list[int], radius: float = ..., remesh_settings: GeotechLocalRemeshRemeshSettings = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechLocalRemeshResult: """Locally remeshes the region around selected seed triangles in a Geological Model Meshes object using configurable remeshing parameters and radius-based region growth.""" ... @@ -5906,7 +5380,7 @@ class _GeotechMakeGmmIntoIntersectedGeometry: min_point: GeotechMakeGmmIntoIntersectedGeometryPoint3D2 = ..., remesh_settings: GeotechMakeGmmIntoIntersectedGeometryRemeshSettings = ..., rotation_angle: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechMakeGmmIntoIntersectedGeometryResult: """Clips a Geological Model Meshes object and processes it to create an intersected geometry mesh.""" ... @@ -5923,7 +5397,7 @@ class _GeotechMakeIntoModelInput: min_point: GeotechMakeIntoModelInputMinPoint = ..., rotation_angle: float = ..., tolerance: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechMakeIntoModelInputResult: """Clips a triangle mesh loosely to the specified bounds, merges duplicate points, and translates the result so it can be used as model input.""" ... @@ -5937,7 +5411,7 @@ class _GeotechMarkVolumesForRemoval: source: ObjectInput, target: GeotechMarkVolumesForRemovalTarget, settings: GeotechMarkVolumesForRemovalSettings, - preview: bool = True, + preview: bool = False, ) -> GeotechMarkVolumesForRemovalResult: """Marks specified volumes in a Geological Model Meshes for removal by setting their material to external, or clears the marking.""" ... @@ -6041,7 +5515,7 @@ class _GeotechParameterizeVolumes: ], include_hard_edges: bool = ..., max_repair_iterations: int = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechParameterizeVolumesResult: """Parameterizes closed volumes from Triangle Mesh or Geological Model Meshes inputs and writes the result as a Design Geometry object.""" ... @@ -6065,7 +5539,7 @@ class _GeotechRemesh: shape_quality_weight: float = ..., strain_tolerance: float = ..., target_h: float = ..., - preview: bool = True, + preview: bool = False, ) -> GeotechRemeshResult: """Remeshes the full Geological Model Meshes object using configurable remeshing parameters to improve triangle quality while preserving the model structure.""" ... @@ -6079,7 +5553,7 @@ class _GeotechRemoveTriangles: source: ObjectInput, target: GeotechRemoveTrianglesTarget, triangle_indices: list[int], - preview: bool = True, + preview: bool = False, ) -> GeotechRemoveTrianglesResult: """Removes the selected triangle indices from a Geological Model Meshes object and uploads the updated mesh as a new object.""" ... @@ -6096,7 +5570,7 @@ class _GeotechSubsumeEmbeddedVolumeInGmm: target_selection: GeotechSubsumeEmbeddedVolumeInGmmExplicitTargetSelection | GeotechSubsumeEmbeddedVolumeInGmmLargestNeighbouringVolumeTargetSelection | GeotechSubsumeEmbeddedVolumeInGmmLargestNeighbouringContactAreaTargetSelection, - preview: bool = True, + preview: bool = False, ) -> GeotechSubsumeEmbeddedVolumeInGmmResult: """Merges a specified source embedded volume into a target volume in a GMM, either by explicit index or by automatic selection strategy.""" ... @@ -6108,7 +5582,7 @@ class _GeotechValidateGmmAsIntersectedGeometry: self, *, source: ObjectInput, - preview: bool = True, + preview: bool = False, ) -> GeotechValidateGmmAsIntersectedGeometryResult: """Checks whether a Geological Model Meshes object is valid as intersected geometry and returns human-readable descriptions of any issues found.""" ... @@ -6712,7 +6186,7 @@ class _IsosurfacingCategory: category: str, target: IsosurfacingCategoryTarget, settings: IsosurfacingCategoryCategorySettings = ..., - preview: bool = True, + preview: bool = False, ) -> IsosurfacingCategoryResult: """Generate closed volume meshes or boundary surfaces from category attributes on 3D grids. Supports single-category extraction.""" ... @@ -6727,7 +6201,7 @@ class _IsosurfacingNumeric: threshold: float, target: IsosurfacingNumericTarget, settings: IsosurfacingNumericNumericSettings = ..., - preview: bool = True, + preview: bool = False, ) -> IsosurfacingNumericResult: """Generate an isosurface at a specified threshold value from numeric attributes on 3D grids. Supports regular, tensor, masked, block model, and unstructured hexahedral and tetrahedral grids.""" ... @@ -6743,7 +6217,7 @@ class _UtilitiesCompress: archive_format: Literal["zip", "zip-stored", "tar", "tar.gz", "tar.bz2", "tar.xz"] = ..., compression_level: int | None = ..., output_name: str = ..., - preview: bool = True, + preview: bool = False, ) -> UtilitiesCompressResult: """Downloads a set of files referenced by File API URLs, bundles them into a single archive (ZIP or TAR with gzip/bzip2/xz compression), uploads the archive back to the File API, and returns a downloadable reference to it.""" ... @@ -6956,7 +6430,7 @@ class ComputeClient: validate: bool | None = ..., deep_validation: bool | None = ..., check_schemas: bool | None = ..., - ) -> TaskResult: + ) -> Any: """Run any task by name, including one this stub does not know about.""" ... def __dir__(self) -> list[str]: ... diff --git a/packages/evo-compute/src/evo/compute/overrides/__init__.py b/packages/evo-compute/src/evo/compute/overrides/__init__.py new file mode 100644 index 000000000..1ead29d9b --- /dev/null +++ b/packages/evo-compute/src/evo/compute/overrides/__init__.py @@ -0,0 +1,88 @@ +# Copyright © 2026 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The override seam: one task at a time, a hand-written runner in place of the generic proxy. + +The generic engine shapes every call from the task's published JSON Schema, which is all +most tasks need. A few want more than the schema can say -- a check that spans two +parameters, a result type with helpers of its own, a signature someone wrote and reviewed. +This is how such a task opts out without the engine growing a special case for it. + +A module at ``evo/compute/overrides//.py`` exposing ``bind`` claims that task:: + + def bind(client: ComputeClient, topic: str, task: str) -> Any: ... + +``client..`` then returns whatever ``bind`` builds instead of the generic +:class:`~evo.compute.engine._TaskProxy`. Named calls through +:meth:`~evo.compute.engine.ComputeClient.arun` call the runner's ``run_parameters`` method, +which shares preparation and checks with its typed ``run`` method. + +The module path is the registration: no decorator, no import of the overrides package from +the engine at start-up, and nothing to keep in step with the catalogue. ```` is the +task name as Python spells it (``normal-score`` -> ``normal_score``), which is also how the +caller spells it on the client. + +An override uses the client's internal generic pipeline, so a job still carries +the same authorization, the same validation and the same reference resolution as any other. +What it replaces is the surface, not the plumbing. +""" + +from __future__ import annotations + +import importlib +import pkgutil +from functools import lru_cache +from pathlib import Path +from types import ModuleType + +__all__ = [ + "load_override", + "overridden_tasks", +] + + +def overridden_tasks() -> list[tuple[str, str]]: + """Every ``(topic, task)`` a hand-written runner claims, as Python spells the names. + + Read from this package rather than from a task catalogue, because an override is a + decision made in code: it applies to whatever the platform advertises under that name, + and it has to be describable without anyone having fetched a catalogue first. + """ + root = Path(__file__).resolve().parent + claimed: list[tuple[str, str]] = [] + for topic in sorted(module.name for module in pkgutil.iter_modules([str(root)]) if module.ispkg): + for task in sorted(module.name for module in pkgutil.iter_modules([str(root / topic)])): + if load_override(topic, task) is not None: + claimed.append((topic, task)) + return claimed + + +@lru_cache(maxsize=None) +def load_override(topic: str, task: str) -> ModuleType | None: + """Import the override module for ``topic``/``task``, or return ``None`` if there is none. + + A :exc:`ModuleNotFoundError` naming a *different* module is re-raised: that is an + override whose own imports are broken, and silently falling back to the generic proxy + would turn a bug into a task that quietly behaves differently. + + Results are cached because this is called on every ``client..`` attribute + access, and the common answer -- no override -- otherwise costs a failed import each + time. Call ``load_override.cache_clear()`` after adding a module at runtime. + """ + module_name = f"{__name__}.{topic}.{task}" + try: + module = importlib.import_module(module_name) + except ModuleNotFoundError as error: + missing = error.name or "" + if module_name == missing or module_name.startswith(f"{missing}."): + return None + raise + return module if hasattr(module, "bind") else None diff --git a/packages/evo-compute/src/evo/compute/overrides/geostatistics/__init__.py b/packages/evo-compute/src/evo/compute/overrides/geostatistics/__init__.py new file mode 100644 index 000000000..f175ccff0 --- /dev/null +++ b/packages/evo-compute/src/evo/compute/overrides/geostatistics/__init__.py @@ -0,0 +1,16 @@ +# Copyright © 2026 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Overrides for tasks in the ``geostatistics`` topic. + +Only the tasks with a module here are overridden; every other geostatistics task is served +by the generic engine. +""" diff --git a/packages/evo-compute/src/evo/compute/overrides/geostatistics/kriging.py b/packages/evo-compute/src/evo/compute/overrides/geostatistics/kriging.py new file mode 100644 index 000000000..184e5e5eb --- /dev/null +++ b/packages/evo-compute/src/evo/compute/overrides/geostatistics/kriging.py @@ -0,0 +1,295 @@ +# Copyright © 2026 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hand-written runner for ``geostatistics/kriging`` -- the worked example of an override. + +Kriging is the task with the most behind it: a parameter model that has been through +review, a result type with helpers a synthesised one could not have, and two mistakes the +published schema has no way to describe. So it is the one that earns a hand-written +surface, and ``client.geostatistics.kriging`` is served from here instead of from the +generic proxy. + +What the override adds, and nothing more: + +* **Checks the JSON Schema cannot express.** The schema bounds each parameter on its own -- + ``min_samples`` and ``max_samples`` are both integers of at least 1 -- but it cannot say + that one must not exceed the other, and it cannot say that a task must not estimate an + attribute from itself. Both are silent failures otherwise: the first finds no + neighbourhood anywhere, the second overwrites its own input. +* **The typed parameter model.** :class:`~evo.compute.tasks.geostatistics.kriging.KrigingParameters` + accepts typed objects and attributes, folds ``source_filter`` and ``target_filter`` into + the source and target the wire expects, and names the arguments as the SDK names them + rather than as the wire spells them. +* **The typed result.** :class:`~evo.compute.tasks.geostatistics.kriging.KrigingResult` -- + ``target_name``, ``attribute_name``, ``schema``, ``get_target_object()``, + ``to_dataframe()`` -- in place of the generic :class:`~evo.compute.outputs.TaskResult`. + +Attribute-based calls and :meth:`~evo.compute.engine.ComputeClient.arun` both use this +runner. The shared engine still discovers, validates, resolves and submits the job. +""" + +from __future__ import annotations + +import dataclasses +import inspect +from typing import TYPE_CHECKING, Any + +from evo.objects import DownloadedObject, ObjectMetadata, ObjectReference +from evo.objects.typed import Attribute, BlockModelAttribute +from evo.objects.typed.base import BaseObject +from pydantic import BaseModel + +from ...exceptions import ParameterValidationError +from ...tasks.common import Filter, SearchNeighborhood, Source, Target +from ...tasks.common.source_target import AnyTypedAttribute, UpdateAttribute +from ...tasks.geostatistics.kriging import ( + BlockDiscretization, + KrigingDiagnostics, + KrigingParameters, + KrigingResult, + KrigingResultModel, + OrdinaryKriging, + SimpleKriging, +) + +if TYPE_CHECKING: + from ...engine import ComputeClient + +__all__ = [ + "KrigingOverride", + "bind", +] + +# The published catalogue uses the British spelling for this parameter. +PARAMETER_ALIASES = {"block_discretisation": "block_discretization"} + +# What each reference parameter takes. The model fields are annotated with their *validated* +# type, which a checker reads as the narrow one; a signature has to say what it accepts. +_SourceInput = Source | Attribute | BlockModelAttribute +_TargetInput = Target | AnyTypedAttribute +_ObjectInput = str | ObjectReference | BaseObject | DownloadedObject | ObjectMetadata + +# Arguments the model folds into another field, and where the wire carries each one. +_FOLDED_ARGUMENTS = { + "source_filter": ("source", "filter"), + "target_filter": ("target", "filter"), + "diagnostics": ("target", "diagnostics"), +} + + +def _declared_fields(model: Any) -> dict[str, str]: + """The keys ``model`` reads, by field name or alias, mapped to the attribute each one fills.""" + if isinstance(model, BaseModel): + keys: dict[str, str] = {} + for name, field in type(model).model_fields.items(): + keys[name] = name + for alias in (field.alias, field.validation_alias): + if isinstance(alias, str): + keys[alias] = name + return keys + if dataclasses.is_dataclass(model) and not isinstance(model, type): + return {field.name: field.name for field in dataclasses.fields(model)} + return {} + + +def _preserve_extra_fields(payload: Any, supplied: Any, model: Any = None) -> None: + """Put back the fields the typed models dropped, so the engine can apply the schema's rule. + + Pydantic ignores keys a model does not declare, so an unknown field would otherwise vanish + before the engine could reject, validate, forward or warn about it. A key the model does + read is not an extra, even when the payload carries it under another name or leaves it out. + """ + if isinstance(supplied, list) and isinstance(payload, list) and len(supplied) == len(payload): + models = model if isinstance(model, list) and len(model) == len(supplied) else [None] * len(supplied) + for payload_item, supplied_item, model_item in zip(payload, supplied, models, strict=True): + _preserve_extra_fields(payload_item, supplied_item, model_item) + return + if not isinstance(supplied, dict) or not isinstance(payload, dict): + return + declared = _declared_fields(model) + for name, value in supplied.items(): + if name in payload: + field = declared.get(name) + _preserve_extra_fields(payload[name], value, getattr(model, field) if field else None) + elif name not in declared: + payload[name] = value + + +def _same_object(left: str, right: str) -> bool: + """Whether two references name the same object, whatever version or spelling each uses. + + Comparing the URLs as text would miss it. :class:`~evo.objects.ObjectReference` is a ``str`` + subclass that keeps whatever the caller wrote, and its path pattern is case-insensitive, so + an uppercase UUID on one side and a lowercase one on the other are one object written two + ways. A trailing fragment does the same. The parsed components are what identify an object; + ``version_id`` is not one of them, since a version names a moment in the object's history. + + A path reference and a UUID reference cannot be told apart without loading the object, so + they never match here -- this guard would rather miss a case than invent one. + """ + first, second = ObjectReference(left), ObjectReference(right) + return (first.environment, first.object_id, first.object_path) == ( + second.environment, + second.object_id, + second.object_path, + ) + + +class KrigingOverride: + """The override's stand-in for the generic task proxy: same ``run(...)``, typed by hand.""" + + params_type = KrigingParameters + + def __init__(self, client: ComputeClient, topic: str, task: str) -> None: + self._client = client + self._topic = topic + self._task = task + + async def run( + self, + *, + source: _SourceInput, + target: _TargetInput, + variogram: _ObjectInput, + neighborhood: SearchNeighborhood, + kriging_method: SimpleKriging | OrdinaryKriging | None = None, + source_filter: Filter | None = None, + target_filter: Filter | None = None, + block_discretization: BlockDiscretization | None = None, + diagnostics: KrigingDiagnostics | None = None, + preview: bool = False, + ) -> KrigingResult: + """Estimate an attribute on a target object by kriging from a source attribute. + + :param source: The object and attribute holding the known values, or the typed + attribute itself. + :param target: Where to write the estimate -- a :class:`~evo.compute.tasks.common.Target`, + or the typed attribute to create or update. + :param variogram: The variogram object, or a reference to it. + :param neighborhood: The search neighbourhood: the ellipsoid to look in, and how many + samples to use. + :param kriging_method: Ordinary kriging by default; :meth:`KrigingMethod.simple` when the + mean is known. + :param source_filter: Restrict the estimate to a subset of the source data. + :param target_filter: Restrict the estimate to a subset of the target object. + :param block_discretization: Sub-block discretization for block kriging. Omitted + means point kriging. + :param diagnostics: Optional diagnostic attributes to write beside the estimate. + :param preview: Opt in to the preview API. Defaults to ``False``. + + :raises ParameterValidationError: If the parameters are inconsistent in a way the + task schema cannot describe, or fail the engine's own checks. + """ + return await self.run_parameters( + dict( + source=source, + target=target, + variogram=variogram, + neighborhood=neighborhood, + kriging_method=kriging_method if kriging_method is not None else OrdinaryKriging(), + source_filter=source_filter, + target_filter=target_filter, + block_discretization=block_discretization, + diagnostics=diagnostics, + preview=preview, + ) + ) + + def _prepare_parameters(self, supplied: dict[str, Any]) -> dict[str, Any]: + parameters = dict(supplied) + preview = parameters.pop("preview", False) + label = f"{self._topic}.{self._task.replace('-', '_')}" + for argument, (name, field) in _FOLDED_ARGUMENTS.items(): + if isinstance(frame := parameters.get(name), dict) and field in frame: + if parameters.get(argument) is not None: + message = f"{argument} was supplied both directly and inside {name}" + raise ParameterValidationError(f"{label}.run(): {message}", task=label, errors=[message]) + frame = dict(frame) + parameters[argument] = frame.pop(field) + parameters[name] = frame + signature = inspect.signature(self.run) + extra = {name: value for name, value in parameters.items() if name not in signature.parameters} + parameters = {name: value for name, value in parameters.items() if name not in extra} + try: + signature.bind(**parameters) + except TypeError as error: + raise ParameterValidationError(f"{label}.run(): {error}", task=label, errors=[str(error)]) from None + validated = parameters + if isinstance(diagnostics := parameters.get("diagnostics"), dict): + # The diagnostics model refuses unknown keys outright; the schema decides what happens to them. + known = {name: value for name, value in diagnostics.items() if name in KrigingDiagnostics.model_fields} + validated = {**parameters, "diagnostics": known} + model = KrigingParameters.model_validate(validated) + self._check_parameters(model) + payload = model.model_dump(mode="json", by_alias=True, exclude_none=True) + _preserve_extra_fields(payload, parameters, model) + for argument, (name, field) in _FOLDED_ARGUMENTS.items(): + _preserve_extra_fields(payload[name].get(field), parameters.get(argument), getattr(model, argument)) + payload.update(extra) + payload["preview"] = preview + return payload + + async def run_parameters( + self, + parameters: dict[str, Any], + *, + validate: bool | None = None, + deep_validation: bool | None = None, + check_schemas: bool | None = None, + ) -> KrigingResult: + """Run named or typed inputs through the same preparation and checks.""" + result = await self._client._run_generic( + self._topic, + self._task, + self._prepare_parameters(parameters), + validate=validate, + deep_validation=deep_validation, + check_schemas=check_schemas, + ) + return KrigingResult(self._client.context, KrigingResultModel.model_validate(dict(result))) + + def _check_parameters(self, parameters: KrigingParameters) -> None: + """Reject the two inconsistencies the task's JSON Schema has no way to state.""" + label = f"{self._topic}.{self._task.replace('-', '_')}" + errors: list[str] = [] + + search = parameters.search + if search.min_samples is not None and search.min_samples > search.max_samples: + errors.append( + f"neighborhood: min_samples ({search.min_samples}) exceeds max_samples " + f"({search.max_samples}), so no location can ever be estimated" + ) + + attribute = parameters.target.attribute + if ( + isinstance(attribute, UpdateAttribute) + and _same_object(parameters.source.object, parameters.target.object) + and attribute.reference == parameters.source.attribute + ): + errors.append( + f"target: {attribute.reference!r} is also the source attribute, " + "so the estimate would overwrite its own input" + ) + + if errors: + raise ParameterValidationError( + f"{label}.run(): " + "; ".join(errors), + task=label, + errors=errors, + ) + + def __repr__(self) -> str: + return f"" + + +def bind(client: ComputeClient, topic: str, task: str) -> KrigingOverride: + """Hand this task to the override. Called by :mod:`evo.compute.overrides`.""" + return KrigingOverride(client, topic, task) diff --git a/packages/evo-compute/src/evo/compute/resolution.py b/packages/evo-compute/src/evo/compute/resolution.py index 9346e9199..c256f3451 100644 --- a/packages/evo-compute/src/evo/compute/resolution.py +++ b/packages/evo-compute/src/evo/compute/resolution.py @@ -69,6 +69,7 @@ _source_from_attribute, _validate_target_attribute, ) +from .validation import _warn_unknown_fields __all__ = [ "ReferenceResolver", @@ -160,9 +161,13 @@ async def resolve( check_schemas=check_schemas, ) properties = schema.get("properties", {}) or {} + additional = schema.get("additionalProperties") + if additional is False: + _warn_unknown_fields(schema, parameters, [], walk.label, root=schema) for name, value in parameters.items(): - node = _deref(properties.get(name, {}), schema) + node = _deref(properties.get(name, additional if isinstance(additional, dict) else {}), schema) walk.resolved[name] = await self._resolve_node(node, value, [name], walk) + _warn_unknown_fields(schema, walk.resolved, [], walk.label, root=schema) return walk.resolved # -- the walk ---------------------------------------------------------- # @@ -172,6 +177,8 @@ async def _resolve_node(self, node: dict[str, Any], value: Any, path: list[str], if value is None: return None node = _deref(node, walk.schema) + if not node: + return value # A tagged union describes several shapes; only the tagged branch's annotations apply. if (discriminator := _discriminator_of(node)) is not None: @@ -195,7 +202,11 @@ async def _resolve_node(self, node: dict[str, Any], value: Any, path: list[str], if not isinstance(value, (dict, AnyTypedAttribute)) and hasattr(value, "model_dump"): value = value.model_dump(mode="json", by_alias=True) - if properties := node.get("properties"): + properties = node.get("properties", {}) or {} + additional = node.get("additionalProperties") + if additional is False and isinstance(value, dict): + _warn_unknown_fields(node, value, path, walk.label, root=walk.schema) + if properties or isinstance(additional, dict): if (frame := self._frame_from_attribute(properties, value, path, walk)) is not None: # Carry on into the walk below rather than returning: the frame's object leaf # has to face ``supported_schemas`` like any other. @@ -209,10 +220,16 @@ async def _resolve_node(self, node: dict[str, Any], value: Any, path: list[str], resolved: dict[str, Any] = {} _graft(walk.resolved, path, resolved) for key, child in value.items(): - child_node = _deref(properties.get(key, {}), walk.schema) + child_node = _deref( + properties.get(key, additional if isinstance(additional, dict) else {}), walk.schema + ) resolved[key] = await self._resolve_node(child_node, child, [*path, key], walk) + _warn_unknown_fields(node, resolved, path, walk.label, root=walk.schema) return resolved + if isinstance(value, dict): + _warn_unknown_fields(node, value, path, walk.label, root=walk.schema) + if node.get("type") == "array" and isinstance(value, list): items = _deref(node.get("items", {}), walk.schema) return [ diff --git a/packages/evo-compute/src/evo/compute/tasks/common/search.py b/packages/evo-compute/src/evo/compute/tasks/common/search.py index b4fd232ad..cb3696bc1 100644 --- a/packages/evo-compute/src/evo/compute/tasks/common/search.py +++ b/packages/evo-compute/src/evo/compute/tasks/common/search.py @@ -16,7 +16,7 @@ from typing import Any from evo.objects.typed.types import Ellipsoid -from pydantic import BaseModel, Field, model_serializer +from pydantic import BaseModel, Field, ValidatorFunctionWrapHandler, field_validator, model_serializer __all__ = [ "SearchNeighborhood", @@ -100,6 +100,20 @@ class SearchNeighborhood(BaseModel): max_drillholes_per_estimate: int | None = Field(default=None, ge=1) """The maximum number of drillholes used in each estimate. Requires a downhole intervals source object.""" + @field_validator("ellipsoid", mode="wrap") + @classmethod + def _ellipsoid_from_wire(cls, value: Any, handler: ValidatorFunctionWrapHandler) -> Ellipsoid: + if not isinstance(value, dict) or "ellipsoid_ranges" not in value: + return handler(value) + if "ranges" in value: + raise ValueError("Supply either ranges or ellipsoid_ranges, not both") + fields = dict(value) + fields["ranges"] = fields.pop("ellipsoid_ranges") + if "rotation" in fields: + fields["_rotation"] = fields.pop("rotation") + ellipsoid = handler(fields) + return Ellipsoid(ranges=ellipsoid.ranges, rotation=ellipsoid.rotation) + @model_serializer def _serialize(self) -> dict[str, Any]: result = { diff --git a/packages/evo-compute/src/evo/compute/validation.py b/packages/evo-compute/src/evo/compute/validation.py index 1a3a50368..cfd181250 100644 --- a/packages/evo-compute/src/evo/compute/validation.py +++ b/packages/evo-compute/src/evo/compute/validation.py @@ -17,12 +17,16 @@ errors: * **Shallow** (always on): required-field presence on the resolved payload. Signature - binding in :mod:`evo.compute.engine` already rejects unknown/missing keyword + binding in :mod:`evo.compute.engine` already rejects missing required keyword arguments; this adds the schema-driven guard that a required field wasn't supplied as ``None`` unless its schema permits null. -* **Deep** (opt-in): full JSON Schema Draft 2020-12 validation, including ``$ref`` / +* **Deep** (opt-in): JSON Schema Draft 2020-12 constraints on known fields, including ``$ref`` / ``$defs`` resolution and discriminated unions, via :mod:`jsonschema`. +Extra fields follow each object's ``additionalProperties`` rule. Closed objects reject +them, schema-valued rules validate them, and explicitly open objects accept them silently. +When the rule is absent, reference resolution warns before forwarding them unchanged. + Reference leaves (properties carrying a ``reference_to`` annotation) are relaxed, because the concrete object/attribute/file shape the schema declares is produced by the reference-resolution layer rather than supplied by the caller. The relaxation is only as @@ -38,6 +42,8 @@ from __future__ import annotations +import logging +import re from collections.abc import Iterator from typing import Any @@ -52,6 +58,8 @@ "validate_parameters", ] +logger = logging.getLogger("compute.validation") + KNOWN_SCHEMA_ANNOTATIONS: frozenset[str] = frozenset( { @@ -383,8 +391,61 @@ def _validate_required(spec: TaskResource, parameters: dict[str, Any], label: st ) +def _warn_unknown_fields( + node: dict[str, Any], + value: dict[str, Any], + path: list[str], + label: str, + *, + root: dict[str, Any] | None = None, +) -> None: + """Check extra fields and warn only when the schema leaves their policy unspecified.""" + properties = node.get("properties") + additional = node.get("additionalProperties") + patterns = node.get("patternProperties", {}) or {} + unknown = sorted( + ( + name + for name in value + if name not in (properties or {}) and not any(re.search(pattern, str(name)) for pattern in patterns) + ), + key=str, + ) + if not unknown or additional is True: + return + location = ".".join(str(part) for part in path) or "parameters" + if additional is False: + message = f"{location}: unexpected parameter(s): {', '.join(repr(name) for name in unknown)}" + raise ParameterValidationError(f"{label}.run(): {message}", task=label, errors=[message]) + if isinstance(additional, dict): + schema = _relax_reference_leaves(node if root is None else root) + validator = Draft202012Validator(schema).evolve( + schema={"type": "object", "additionalProperties": _relax_reference_leaves(additional)} + ) + errors = sorted( + validator.iter_errors({name: value[name] for name in unknown}), + key=lambda error: [str(part) for part in error.absolute_path], + ) + if errors: + prefix = f"{location}." if path else "" + messages = [f"{prefix}{_describe(error)}" for error in errors] + raise ParameterValidationError( + f"{label}.run(): parameters do not match the task schema:\n - " + "\n - ".join(messages), + task=label, + errors=messages, + ) + return + logger.warning( + "%s: the task catalogue does not describe these fields at %s: %s. " + "Their values are to be submitted as is. Check the field names and supply values ready for the service.", + label, + location, + ", ".join(repr(name) for name in unknown), + ) + + def _validate_deep(spec: TaskResource, parameters: dict[str, Any], label: str) -> None: - """Deep check: validate the payload against the task schema (reference leaves relaxed).""" + """Check the published constraints, relaxing only reference input shapes.""" schema = _relax_reference_leaves(spec.parameters or {}) validator = Draft202012Validator(schema) # Rendered parts keep the key uniformly typed: raw paths mix str and int and won't sort. @@ -410,7 +471,7 @@ def validate_parameters( :param spec: The discovered task, carrying its ``parameters`` JSON Schema. :param parameters: The resolved payload about to be submitted. - :param deep: When ``True``, also run full JSON Schema Draft 2020-12 validation. + :param deep: When ``True``, check the full JSON Schema Draft 2020-12 constraints. :param task_label: A ``topic.task`` label for error messages; derived from ``spec`` when omitted. diff --git a/packages/evo-compute/stubs/README.md b/packages/evo-compute/stubs/README.md index 57210b71f..53bc22a06 100644 --- a/packages/evo-compute/stubs/README.md +++ b/packages/evo-compute/stubs/README.md @@ -106,6 +106,11 @@ exactly what was taken and from where, and `tests/test_schema_conformance.py::test_the_snapshot_still_describes_tasks_the_catalogue_advertises` reports drift against a live catalogue when run with credentials. +**Custom runners supply their own types.** An overridden task imports its runner class +instead of generating a second signature. Attribute-based calls and `arun(...)` both use +that runner's argument names, checks and native result. Overrides are found in the package, +so they remain available even if the catalogue snapshot does not mention them. + **Results are `TaskResult` / `ResultNode` subclasses, not `TypedDict`s.** The engine hydrates what the platform sends into nodes that are still dictionaries but additionally load what their references point at. Generating the result tree as classes is what makes @@ -145,7 +150,7 @@ The stub is one half of a contract the engine enforces at run time from the same | Mistake | Caught statically by | Caught at run time by | |---|---|---| | unknown topic or task | *nothing — by design* | discovery lookup in `arun` | -| unknown parameter | the stub's `run(...)` signature | signature binding at the top level; JSON Schema rules under deep validation | +| unknown parameter | the stub's `run(...)` signature | the object's `additionalProperties` rule | | missing required parameter | the stub's `run(...)` signature | signature binding, then `validate_parameters` | | wrong scalar type, bad enum member, missing nested field | the stub's `TypedDict`s and `Literal`s | `validate_parameters(..., deep=True)` | | a reference nothing could resolve | the stub's `ObjectInput` / `AttributeInput` unions | `ReferenceResolver`, then `validate_parameters(..., deep=True)` | @@ -156,6 +161,19 @@ pass. `tests/test_stubgen.py::TestDeepValidationAgreement` asserts the two halve the payload from `usage_ok.py` passes deep validation, and its known-value and reference mistakes are rejected at run time too. +Extra fields follow the rule published for each object: + +- `additionalProperties: false` rejects extra fields locally. +- A schema in `additionalProperties` validates each extra value, even without deep validation. +- `additionalProperties: true` accepts extra fields without a warning. +- An omitted rule forwards extra fields with a warning that their values will be submitted as is. + +These rules also apply to optional settings on SDK models, such as neighborhood octant +and drillhole limits. Unknown values must already be ready for the service; the engine +cannot resolve references without a schema. Deep validation retains closed-object and +`unevaluatedProperties` constraints instead of relaxing them. Preview is always an +explicit opt-in, including for tasks with a feature flag. + ## Verified with | Checker | Result | diff --git a/packages/evo-compute/stubs/checks/usage_ok.py b/packages/evo-compute/stubs/checks/usage_ok.py index 47685b8e2..2af607c92 100644 --- a/packages/evo-compute/stubs/checks/usage_ok.py +++ b/packages/evo-compute/stubs/checks/usage_ok.py @@ -15,17 +15,13 @@ let ``tests/test_stubgen.py`` do it when the checker is installed. """ -from typing import TYPE_CHECKING - from evo.common import IContext from evo.objects.typed import Attribute, BaseObject, PendingAttribute from typing_extensions import assert_type from evo.compute import ComputeClient from evo.compute.tasks.common import Ellipsoid, EllipsoidRanges, Rotation, SearchNeighborhood - -if TYPE_CHECKING: - from evo.compute.engine import GeostatisticsKrigingResult +from evo.compute.tasks.geostatistics.kriging import KrigingMethod, KrigingResult def _model_neighborhood() -> SearchNeighborhood: @@ -149,8 +145,8 @@ async def typed_attribute_source(context: IContext, grade: Attribute, kriged: Pe ellipsoid=Ellipsoid(ranges=EllipsoidRanges(major=100.0, semi_major=100.0, minor=50.0)), max_samples=20, ), - kriging_method={"type": "ordinary"}, + kriging_method=KrigingMethod.ORDINARY, ) - assert_type(result, GeostatisticsKrigingResult) - print(result.target.attribute.name) - await result.target.load() + assert_type(result, KrigingResult) + print(result.target_name, result.attribute_name) + await result.get_target_object() diff --git a/packages/evo-compute/tests/test_behavior_parity.py b/packages/evo-compute/tests/test_behavior_parity.py index dcae4a5eb..a7e2730f9 100644 --- a/packages/evo-compute/tests/test_behavior_parity.py +++ b/packages/evo-compute/tests/test_behavior_parity.py @@ -137,8 +137,7 @@ def _runner_call(self, runner_cls, inputs: dict[str, Any]): return lambda: runner_cls(self.context, runner_cls.params_type(**inputs)) def _engine_call(self, runner_cls, inputs: dict[str, Any]): - task = getattr(self.client.geostatistics, runner_cls.task.replace("-", "_")) - return lambda: task.run(**inputs) + return lambda: self.client._run_generic("geostatistics", runner_cls.task, inputs) def _catalogue(self, runner_cls): """Serve the engine the task's schema, derived from the runner's own parameter model.""" @@ -444,14 +443,8 @@ async def test_a_source_without_an_attribute(self) -> None: class TestWhereTheTwoPathsDiffer(BehaviorParityTestCase): """The cases the contract above does not cover, asserted as they are rather than left implied.""" - async def test_a_misspelled_parameter_reaches_the_platform_from_the_runner_only(self) -> None: - """DIVERGENCE, the engine is stricter: a parameter model ignores what it does not recognise. - - None of these models set ``extra="forbid"``, so a typo is dropped in silence and the - job runs with a default the caller did not intend. The engine binds the call to a - signature synthesised from the schema, so the same typo is a refusal -- and one that - does not depend on validation being switched on. - """ + async def test_an_extra_parameter_warns_when_the_schema_leaves_its_policy_unspecified(self) -> None: + """The legacy model ignores extra fields; the engine warns and preserves them.""" misspelled = declustering_inputs(powr=2.5) del misspelled["power"] @@ -459,8 +452,11 @@ async def test_a_misspelled_parameter_reaches_the_platform_from_the_runner_only( self.assertNotIn("powr", payload) self.assertEqual(2.0, payload["power"], "the runner should have fallen back to its own default") - error = await self.engine_refusal(DeclusteringRunner, **misspelled) - self.assertIn("got an unexpected keyword argument 'powr'", str(error)) + with self.assertLogs("compute.validation", level="WARNING") as logs: + payload = await self.assertEngineAccepts(DeclusteringRunner, **misspelled) + self.assertEqual(2.5, payload["powr"]) + self.assertNotIn("power", payload) + self.assertTrue(any("'powr'" in message for message in logs.output)) async def test_a_value_the_model_coerces_is_refused_by_the_engine(self) -> None: """DIVERGENCE, the engine is stricter: pydantic reads ``"500"`` as ``500``; JSON Schema does not. @@ -502,14 +498,16 @@ async def test_value_checks_are_opt_in_on_the_engine(self) -> None: await self.engine_refusal(ConSimRunner, **too_many) async def test_the_default_engine_still_refuses_the_mistakes_it_can_see(self) -> None: - """Deep validation buys value checks, not the whole of them: three refusals survive without it.""" + """Required parameters and resolvable references are always checked.""" with self.engine(): # the default missing = consim_inputs() del missing["variogram_model"] self.assertBlames("engine", await self.engine_refusal(ConSimRunner, **missing), "variogram_model") unknown = consim_inputs(number_of_liness=500) - self.assertBlames("engine", await self.engine_refusal(ConSimRunner, **unknown), "number_of_liness") + with self.assertLogs("compute.validation", level="WARNING"): + payload = await self.assertEngineAccepts(ConSimRunner, **unknown) + self.assertEqual(500, payload["number_of_liness"]) unresolvable = consim_inputs(variogram_model="variogram.json") self.assertBlames("engine", await self.engine_refusal(ConSimRunner, **unresolvable), "variogram_model") diff --git a/packages/evo-compute/tests/test_engine.py b/packages/evo-compute/tests/test_engine.py index 337640aa4..ee9bb5b22 100644 --- a/packages/evo-compute/tests/test_engine.py +++ b/packages/evo-compute/tests/test_engine.py @@ -36,8 +36,6 @@ def _object_url(suffix: str) -> str: SOURCE_URL = _object_url("10") -TARGET_URL = _object_url("20") -VARIOGRAM_URL = _object_url("30") FILE_URL = "https://unittest.localhost/file/v2/orgs/00000000-0000-0000-0000-000000000001/files/mesh" @@ -160,19 +158,22 @@ async def test_parameters_that_cannot_all_be_keywords_are_taken_as_given(self) - self.assertEqual(inspect.Parameter.VAR_KEYWORD, parameters["parameters"].kind) async def test_fallback_parameters_still_reject_missing_and_unexpected_names(self) -> None: - """Fallback ``**parameters`` calls still check required and unknown schema keys.""" - self.catalogue["results"] += [ - _task("mesh-tools", "export", {"from": {"type": "string"}}, required=["from"]), - _task( - "mesh-tools", - "merge", - {"user-info": {"type": "string"}, "user_info": {"type": "string"}}, - required=["user-info"], - ), - ] + """Fallback ``**parameters`` calls still check required and closed-object schema keys.""" + export_spec = _task("mesh-tools", "export", {"from": {"type": "string"}}, required=["from"]) + export_spec["parameters"]["additionalProperties"] = False + merge_spec = _task( + "mesh-tools", + "merge", + {"user-info": {"type": "string"}, "user_info": {"type": "string"}}, + required=["user-info"], + ) + merge_spec["parameters"]["additionalProperties"] = False + self.catalogue["results"] += [export_spec, merge_spec] with self.catalogue_response(), self.mock_job_client() as submit: with self.assertRaisesRegex(ParameterValidationError, r"mesh_tools\.export\.run\(\):.*'from'"): await self.client.mesh_tools.export.run() + with self.assertRaisesRegex(ParameterValidationError, r"mesh-tools\.export\.run\(\):.*'from'"): + await self.client.arun("mesh-tools", "export", {}, validate=False) with self.assertRaisesRegex(ParameterValidationError, r"mesh_tools\.export\.run\(\):.*'bogus'"): await self.client.mesh_tools.export.run(**{"from": "mesh-1", "bogus": 1}) with self.assertRaisesRegex(ParameterValidationError, r"mesh-tools\.merge\.run\(\):.*'user-info'"): @@ -192,7 +193,7 @@ async def test_catalogue_is_cached_across_runs(self) -> None: """The catalogue is fetched once and reused for later runs (any task/topic).""" with self.catalogue_response(), self.mock_job_client(): await self.client.geostatistics.declustering.run(source=SOURCE_URL) - await self.client.geostatistics.kriging.run(source=SOURCE_URL, target=TARGET_URL, variogram=VARIOGRAM_URL) + await self.client.geostatistics.normal_score.run(distribution=SOURCE_URL) self.assertEqual(1, self.transport.request.call_count) async def test_catalogue_is_shared_across_topics(self) -> None: @@ -245,18 +246,21 @@ async def test_explicit_none_is_forwarded(self) -> None: async def test_preview_is_off_by_default_even_for_feature_flagged_tasks(self) -> None: with self.catalogue_response(), self.mock_job_client() as submit: - await self.client.geostatistics.kriging.run(source=SOURCE_URL, target=TARGET_URL, variogram=VARIOGRAM_URL) + await self.client.converter.obj_import.run(file=FILE_URL) self.assertFalse(submit.await_args.kwargs["preview"]) - async def test_preview_can_be_enabled(self) -> None: - """The preview flag is passed to the job, not sent as a task parameter.""" + async def test_preview_can_be_overridden(self) -> None: + """Explicit preview settings do not become task parameters.""" with self.catalogue_response(), self.mock_job_client() as submit: - await self.client.geostatistics.kriging.run( - source=SOURCE_URL, target=TARGET_URL, variogram=VARIOGRAM_URL, preview=True - ) - self.assertTrue(submit.await_args.kwargs["preview"]) + await self.client.converter.obj_import.run(file=FILE_URL, preview=False) + self.assertFalse(submit.await_args.kwargs["preview"]) self.assertNotIn("preview", submit.await_args.kwargs["parameters"]) + async def test_preview_can_be_explicitly_enabled(self) -> None: + with self.catalogue_response(), self.mock_job_client() as submit: + await self.client.converter.obj_import.run(file=FILE_URL, preview=True) + self.assertTrue(submit.await_args.kwargs["preview"]) + # -- errors ------------------------------------------------------------ # async def test_unknown_task_raises_attribute_error(self) -> None: @@ -275,15 +279,69 @@ async def test_missing_required_parameter_raises_validation_error(self) -> None: submit.assert_not_awaited() async def test_unexpected_parameter_raises_validation_error(self) -> None: - """An unknown keyword argument is rejected, and the error names the task.""" + spec = next(task for task in self.catalogue["results"] if task["name"] == "declustering") + spec["parameters"]["additionalProperties"] = False with self.catalogue_response(), self.mock_job_client() as submit: with self.assertRaises(ParameterValidationError) as ctx: await self.client.geostatistics.declustering.run(source=SOURCE_URL, bogus=1) self.assertIn("geostatistics.declustering.run():", str(ctx.exception)) self.assertIn("bogus", str(ctx.exception)) - # Binding failures populate ``errors`` too, so callers never have to parse the message. - self.assertEqual(["got an unexpected keyword argument 'bogus'"], ctx.exception.errors) + self.assertEqual(["parameters: unexpected parameter(s): 'bogus'"], ctx.exception.errors) + submit.assert_not_awaited() + + async def test_unspecified_extra_field_policy_warns_and_preserves_values(self) -> None: + spec = next(task for task in self.catalogue["results"] if task["name"] == "declustering") + spec["parameters"].pop("additionalProperties", None) + extra = {"new_option": [1, None, {"enabled": True}]} + with ( + self.catalogue_response(), + self.mock_job_client() as submit, + self.assertLogs("compute.validation", level="WARNING") as logs, + ): + await self.client.geostatistics.declustering.run(source=SOURCE_URL, new_parameter=extra) + self.assertEqual(extra, submit.await_args.kwargs["parameters"]["new_parameter"]) + self.assertEqual(1, len(logs.output)) + self.assertIn("to be submitted as is", logs.output[0]) + + async def test_explicitly_open_extra_fields_do_not_warn(self) -> None: + spec = next(task for task in self.catalogue["results"] if task["name"] == "declustering") + spec["parameters"]["additionalProperties"] = True + with ( + self.catalogue_response(), + self.mock_job_client() as submit, + self.assertNoLogs("compute.validation", level="WARNING"), + ): + await self.client.geostatistics.declustering.run(source=SOURCE_URL, new_parameter=2) + self.assertEqual(2, submit.await_args.kwargs["parameters"]["new_parameter"]) + + async def test_typed_extra_fields_are_validated_without_deep_validation(self) -> None: + spec = next(task for task in self.catalogue["results"] if task["name"] == "declustering") + spec["parameters"]["additionalProperties"] = {"type": "integer", "minimum": 1} + with self.catalogue_response(), self.mock_job_client() as submit: + with self.assertRaisesRegex(ParameterValidationError, "new_count"): + await self.client.geostatistics.declustering.run(source=SOURCE_URL, new_count="invalid") + submit.assert_not_awaited() + await self.client.geostatistics.declustering.run(source=SOURCE_URL, new_count=2) + self.assertEqual(2, submit.await_args.kwargs["parameters"]["new_count"]) + + async def test_unsupported_optional_neighborhood_settings_are_rejected_locally(self) -> None: + from evo.compute import _stubgen + from test_payload_parity import _search + + captured = next( + task for task in _stubgen.load_snapshot(_stubgen.DEFAULT_SNAPSHOT_DIR) if task.name == "declustering" + ) + neighborhood = captured.parameters["properties"]["neighborhood"] + neighborhood["additionalProperties"] = False + neighborhood["properties"].pop("max_samples_per_octant", None) + spec = next(task for task in self.catalogue["results"] if task["name"] == "declustering") + spec["parameters"]["properties"]["neighborhood"] = neighborhood + with self.catalogue_response(), self.mock_job_client() as submit: + with self.assertRaisesRegex(ParameterValidationError, "neighborhood.*max_samples_per_octant"): + await self.client.geostatistics.declustering.run( + source=SOURCE_URL, neighborhood=_search().model_copy(update={"max_samples_per_octant": 3}) + ) submit.assert_not_awaited() # -- progressive ergonomics (cache-backed, no forced discovery) -------- # diff --git a/packages/evo-compute/tests/test_overrides.py b/packages/evo-compute/tests/test_overrides.py new file mode 100644 index 000000000..cbdbeb86d --- /dev/null +++ b/packages/evo-compute/tests/test_overrides.py @@ -0,0 +1,534 @@ +# Copyright © 2026 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The override seam and the Kriging runner that uses it. + +Three things are worth holding an override to, and they are the three sections below. + +It has to be reached. The module path is the only registration, so what matters is that +``client.geostatistics.kriging`` is the hand-written runner while every other task -- +and the same task through ``arun`` -- use the same runner. + +It has to earn its place. An override exists to do what the generic path cannot, so each +addition is pinned by a test that fails if the check or the typed result were dropped. + +And it has to send the same job. That is the part an override can quietly get wrong, so the +last section runs one set of inputs through all three paths -- the hand-written task runner, +the override, and the generic engine -- and requires ``JobClient.submit`` to have been +handed the same payload by each, equal in value and in concrete type. The +:mod:`test_payload_parity` harness already makes exactly that comparison for the runner and +the engine; this file adds the override as a third path through the same machinery rather +than restating it. +""" + +from __future__ import annotations + +import inspect +import json +from collections.abc import Iterator +from contextlib import contextmanager +from types import SimpleNamespace +from typing import Any +from unittest import mock + +from evo.compute import ComputeClient, ParameterValidationError, TaskResource, _stubgen +from evo.compute.discovery import DiscoveryClient +from evo.compute.engine import _TaskProxy +from evo.compute.overrides import load_override +from evo.compute.overrides.geostatistics.kriging import KrigingOverride +from evo.compute.tasks import CreateAttribute, Source, Target, UpdateAttribute +from evo.compute.tasks.common import Filter, FilterCondition +from evo.compute.tasks.geostatistics.kriging import ( + BlockDiscretization, + KrigingMethod, + KrigingResult, + KrigingRunner, +) +from test_payload_parity import ( + GRADE_ATTRIBUTE, + POINTSET_URL, + TARGET_URL, + VARIOGRAM_URL, + PayloadParityTestCase, + _capture_submit, + _pending_attribute, + _search, + _SubmitCaptured, + payload_differences, + task_spec, +) + +TASK_RESULT = { + "message": "Kriging complete.", + "target": { + "reference": TARGET_URL, + "name": "Block model", + "schema_id": "/objects/block-model/1.2.0/block-model.schema.json", + "attribute": {"reference": "attributes[?name=='kriged_grade']", "name": "kriged_grade"}, + }, +} + + +def kriging_catalogue(**overrides: Any) -> list[TaskResource]: + """The kriging schema as the catalogue advertises it. + + The runner, the override and the catalogue all name this task ``kriging``, so the + comparison is about the payload rather than about which name each path happens to use. + """ + return [task_spec(KrigingRunner).model_copy(update=overrides)] + + +def published_catalogue(node: str | None = None, rule: Any = None) -> list[TaskResource]: + """Kriging as the captured catalogue publishes it, with the extra-field rule at ``node`` set to ``rule``. + + ``node`` is a dotted path through ``properties``; a ``rule`` of ``None`` removes + ``additionalProperties`` there, leaving the rule unspecified. + """ + path = _stubgen.DEFAULT_SNAPSHOT_DIR / "geostatistics" / "kriging.json" + task = TaskResource.model_validate( + {**json.loads(path.read_text(encoding="utf-8")), "topic": "geostatistics", "name": "kriging"} + ) + if node: + schema = task.parameters + for part in node.split("."): + schema = schema["properties"][part] + if rule is None: + schema.pop("additionalProperties", None) + else: + schema["additionalProperties"] = rule + return [task] + + +class OverrideTestCase(PayloadParityTestCase): + """A client whose discovery is mocked, and the inputs every case here starts from.""" + + def setUp(self) -> None: + super().setUp() + self.catalogue = mock.patch.object( + DiscoveryClient, "list_tasks", mock.AsyncMock(return_value=kriging_catalogue()) + ) + + def inputs(self, **overrides: Any) -> dict[str, Any]: + """One set of inputs, named as the override names them.""" + inputs: dict[str, Any] = dict( + source=Source(object=POINTSET_URL, attribute=GRADE_ATTRIBUTE), + target=Target(object=TARGET_URL, attribute=CreateAttribute(name="kriged_grade")), + variogram=VARIOGRAM_URL, + neighborhood=_search(), + kriging_method=KrigingMethod.ORDINARY, + ) + inputs.update(overrides) + return inputs + + @contextmanager + def completed_job(self, results: dict) -> Iterator[mock.AsyncMock]: + """Let a submission through and hand back ``results``, so the typed result gets built.""" + job = mock.Mock(wait_for_results=mock.AsyncMock(return_value=results)) + submit = mock.AsyncMock(return_value=job) + with self.catalogue, mock.patch("evo.compute.engine.JobClient") as job_client: + job_client.submit = submit + yield submit + + async def override_payload(self, **inputs: Any) -> dict[str, Any]: + """The payload the override submits for ``inputs``.""" + with self.catalogue, _capture_submit("evo.compute.engine") as submit: + with self.assertRaises(_SubmitCaptured): + await self.client.geostatistics.kriging.run(**inputs) + return submit.await_args.kwargs["parameters"] + + +# --------------------------------------------------------------------------- # +# The seam +# --------------------------------------------------------------------------- # + + +class TestTheSeam(OverrideTestCase): + def test_an_overridden_task_is_served_by_its_own_runner(self) -> None: + """The module at ``overrides/geostatistics/kriging.py`` claims the task.""" + self.assertIsInstance(self.client.geostatistics.kriging, KrigingOverride) + + async def test_published_kriging_spelling_is_used_on_the_wire(self) -> None: + catalogue = mock.AsyncMock(return_value=published_catalogue()) + with ( + mock.patch.object(DiscoveryClient, "list_tasks", catalogue), + _capture_submit("evo.compute.engine") as submit, + ): + with self.assertRaises(_SubmitCaptured): + await self.client.geostatistics.kriging.run( + **self.inputs(block_discretization=BlockDiscretization(nx=2, ny=3, nz=4)) + ) + + payload = submit.await_args.kwargs["parameters"] + self.assertEqual({"nx": 2, "ny": 3, "nz": 4}, payload["block_discretisation"]) + self.assertNotIn("block_discretization", payload) + catalogue.assert_awaited_once() + + def test_the_override_remains_async_without_a_blocking_wrapper(self) -> None: + self.assertTrue(inspect.iscoroutinefunction(self.client.geostatistics.kriging.run)) + + def test_every_other_task_stays_generic(self) -> None: + """An override is one task opting out, not a change to how tasks are reached.""" + self.assertIsInstance(self.client.geostatistics.declustering, _TaskProxy) + self.assertIsInstance(self.client.geostatistics.normal_score, _TaskProxy) + self.assertIsInstance(self.client.converter.obj_import, _TaskProxy) + + def test_reaching_a_task_still_costs_no_discovery(self) -> None: + """``bind`` is handed the client, not a spec, so the seam keeps attribute access free.""" + list_tasks = mock.AsyncMock(return_value=kriging_catalogue()) + with mock.patch.object(DiscoveryClient, "list_tasks", list_tasks): + _ = self.client.geostatistics.kriging + list_tasks.assert_not_awaited() + + def test_a_task_is_keyed_as_the_caller_spells_it(self) -> None: + """A hyphenated task is reached with underscores, so that is where its module lives.""" + self.assertIsNotNone(load_override("geostatistics", "kriging")) + self.assertIsNone(load_override("geostatistics", "normal-score")) + self.assertIsNone(load_override("geostatistics", "declustering")) + self.assertIsNone(load_override("no-such-topic", "no_such_task")) + + def test_a_module_without_bind_does_not_claim_its_task(self) -> None: + """``bind`` is the contract; a module that does not expose it is not an override.""" + with mock.patch("importlib.import_module", return_value=SimpleNamespace()): + self.assertIsNone(load_override.__wrapped__("geostatistics", "declustering")) + + def test_a_broken_override_is_not_silently_skipped(self) -> None: + """An override whose own imports fail is a bug, not an absent override.""" + error = ModuleNotFoundError("No module named 'somethingelse'", name="somethingelse") + with mock.patch("importlib.import_module", side_effect=error): + with self.assertRaises(ModuleNotFoundError): + load_override.__wrapped__("geostatistics", "kriging") + + async def test_arun_cannot_bypass_the_override_checks(self) -> None: + with self.catalogue, _capture_submit("evo.compute.engine") as submit: + with self.assertRaises(ParameterValidationError): + await self.client.arun( + "geostatistics", + "kriging", + self.inputs(neighborhood=_search(min_samples=99)), + ) + submit.assert_not_awaited() + + +# --------------------------------------------------------------------------- # +# What the override adds +# --------------------------------------------------------------------------- # + + +class TestWhatTheOverrideAdds(OverrideTestCase): + async def test_named_extra_fields_are_preserved_when_the_schema_leaves_the_rule_unspecified(self) -> None: + catalogue = kriging_catalogue() + catalogue[0].parameters.pop("additionalProperties", None) + self.catalogue = mock.patch.object(DiscoveryClient, "list_tasks", mock.AsyncMock(return_value=catalogue)) + with self.completed_job(TASK_RESULT) as submit, self.assertLogs("compute.validation", level="WARNING") as logs: + await self.client.arun("geostatistics", "kriging", {**self.inputs(), "new_input": {"enabled": True}}) + self.assertEqual({"enabled": True}, submit.await_args.kwargs["parameters"]["new_input"]) + self.assertTrue(any("'new_input'" in message for message in logs.output)) + + async def test_named_extra_fields_are_rejected_when_the_schema_closes_the_object(self) -> None: + with self.completed_job(TASK_RESULT) as submit: + with self.assertRaisesRegex(ParameterValidationError, "new_input"): + await self.client.arun("geostatistics", "kriging", {**self.inputs(), "new_input": True}) + submit.assert_not_awaited() + + async def test_arun_returns_the_same_native_result_as_run(self) -> None: + parameters = KrigingRunner.params_type( + source=self.inputs()["source"], + target=self.inputs()["target"], + variogram=VARIOGRAM_URL, + neighborhood=_search(), + ) + with self.completed_job(TASK_RESULT) as submit: + result = await self.client.arun( + "geostatistics", + "kriging", + parameters.model_dump(mode="json", by_alias=True, exclude_none=True), + ) + self.assertIsInstance(result, KrigingResult) + self.assertEqual( + parameters.model_dump(mode="json", by_alias=True, exclude_none=True)["neighborhood"], + submit.await_args.kwargs["parameters"]["neighborhood"], + ) + + def test_neighborhood_reads_its_serialized_shape_without_losing_rotation(self) -> None: + neighborhood = _search().model_copy(update={"max_samples_per_octant": 3}) + restored = type(neighborhood).model_validate(neighborhood.model_dump(mode="json")) + self.assertEqual(neighborhood, restored) + + async def test_valid_parameters_are_accepted(self) -> None: + """Anti-vacuous guard: the checks below must reject their own case, not every case.""" + with self.completed_job(TASK_RESULT) as submit: + await self.client.geostatistics.kriging.run(**self.inputs()) + submit.assert_awaited_once() + + async def test_a_search_that_can_never_be_satisfied_is_refused(self) -> None: + """Each bound is legal on its own; the schema has no way to say they must agree.""" + with self.catalogue, _capture_submit("evo.compute.engine") as submit: + with self.assertRaises(ParameterValidationError) as caught: + await self.client.geostatistics.kriging.run(**self.inputs(neighborhood=_search(min_samples=99))) + + self.assertIn("min_samples (99) exceeds max_samples (20)", str(caught.exception)) + self.assertEqual("geostatistics.kriging", caught.exception.task) + submit.assert_not_awaited() + + async def test_estimating_an_attribute_from_itself_is_refused(self) -> None: + """Source and target are each valid references; pointing them at one attribute is not.""" + target = Target(object=POINTSET_URL, attribute=UpdateAttribute(reference=GRADE_ATTRIBUTE)) + with self.catalogue, _capture_submit("evo.compute.engine") as submit: + with self.assertRaises(ParameterValidationError) as caught: + await self.client.geostatistics.kriging.run(**self.inputs(target=target)) + + self.assertIn("also the source attribute", str(caught.exception)) + submit.assert_not_awaited() + + async def test_updating_a_different_attribute_on_the_source_object_is_allowed(self) -> None: + """The check is attribute-level: writing an estimate back onto the source object is fine.""" + target = Target(object=POINTSET_URL, attribute=UpdateAttribute(reference="attributes[?name=='estimate']")) + with self.completed_job(TASK_RESULT) as submit: + await self.client.geostatistics.kriging.run(**self.inputs(target=target)) + submit.assert_awaited_once() + + async def test_a_versioned_source_url_is_still_the_same_object(self) -> None: + """``?version=`` names a moment in an object's history, not a different object.""" + source = Source(object=f"{POINTSET_URL}?version=42", attribute=GRADE_ATTRIBUTE) + target = Target(object=POINTSET_URL, attribute=UpdateAttribute(reference=GRADE_ATTRIBUTE)) + with self.catalogue, _capture_submit("evo.compute.engine") as submit: + with self.assertRaises(ParameterValidationError): + await self.client.geostatistics.kriging.run(**self.inputs(source=source, target=target)) + submit.assert_not_awaited() + + async def test_the_same_object_written_a_different_way_is_still_the_same_object(self) -> None: + """``ObjectReference`` keeps the caller's spelling, so the guard cannot compare text. + + The reference pattern is case-insensitive about the UUIDs it matches, and a fragment + never reaches the path at all, so both of these are the source object under another name. + """ + object_id = "0123abcd-ef00-0000-0000-00000000beef" + canonical = f"{POINTSET_URL.rsplit('/', 1)[0]}/{object_id}" + for spelling in (canonical.replace(object_id, object_id.upper()), f"{canonical}#section"): + with self.subTest(spelling=spelling): + source = Source(object=spelling, attribute=GRADE_ATTRIBUTE) + target = Target(object=canonical, attribute=UpdateAttribute(reference=GRADE_ATTRIBUTE)) + with self.catalogue, _capture_submit("evo.compute.engine") as submit: + with self.assertRaises(ParameterValidationError): + await self.client.geostatistics.kriging.run(**self.inputs(source=source, target=target)) + submit.assert_not_awaited() + + async def test_the_same_attribute_on_a_different_object_is_allowed(self) -> None: + """The other direction: the same expression against another object is not a collision.""" + target = Target(object=TARGET_URL, attribute=UpdateAttribute(reference=GRADE_ATTRIBUTE)) + with self.completed_job(TASK_RESULT) as submit: + await self.client.geostatistics.kriging.run(**self.inputs(target=target)) + submit.assert_awaited_once() + + async def test_the_result_is_the_typed_one(self) -> None: + """A hand-curated result with helpers the generic ``TaskResult`` could not synthesise.""" + with self.completed_job(TASK_RESULT): + result = await self.client.geostatistics.kriging.run(**self.inputs()) + + self.assertIsInstance(result, KrigingResult) + self.assertEqual("Kriging complete.", result.message) + self.assertEqual("Block model", result.target_name) + self.assertEqual("kriged_grade", result.attribute_name) + self.assertEqual("block-model", result.schema.sub_classification) + + def test_the_signature_is_the_hand_written_one(self) -> None: + """Not synthesised from the schema: the arguments read as the typed SDK names them.""" + parameters = inspect.signature(self.client.geostatistics.kriging.run).parameters + self.assertIn("neighborhood", parameters) + self.assertIn("kriging_method", parameters) + self.assertNotIn("search", parameters) + self.assertNotIn("method", parameters) + self.assertIn("source_filter", parameters) # folded into ``source``; not a task parameter + self.assertNotIn("self", parameters) + + +class TestFoldedInputsFollowTheExtraFieldRules(OverrideTestCase): + """Filters and diagnostics are folded into ``source`` and ``target``, so their extra fields must survive the fold. + + The typed models drop a key they do not declare, which would leave the schema's rule for + that object nothing to act on. These run against the captured catalogue, with the rule of + the object that carries the folded input switched in turn. + """ + + CONDITION = {"type": "condition", "attribute": GRADE_ATTRIBUTE, "operator": "greater_than", "threshold": 0.5} + + RULES = { + "closed": (False, "rejected"), + "open": (True, "forwarded"), + "typed, matching value": ({"type": "integer"}, "forwarded"), + "typed, mismatching value": ({"type": "string"}, "rejected"), + "unspecified": (None, "warned"), + } + + def setUp(self) -> None: + super().setUp() + self.client = ComputeClient(self.context, check_schemas=False) + + def folded_inputs(self) -> dict[str, tuple[str, dict[str, Any]]]: + """Each way of supplying a folded input, by the object whose rule applies to it.""" + target_condition = {"type": "condition", "attribute": "attributes[?name=='domain']", "operator": "in"} + source_frame = {"object": POINTSET_URL, "attribute": GRADE_ATTRIBUTE} + return { + "source_filter": ("source.filter", {"source_filter": {"where": self.CONDITION, "extra": 7}}), + "source.filter": ( + "source.filter", + {"source": {**source_frame, "filter": {"where": self.CONDITION, "extra": 7}}}, + ), + "target_filter": ( + "target.filter", + {"target_filter": {"where": {**target_condition, "values": ["LMS1"]}, "extra": 7}}, + ), + "diagnostics": ("target.diagnostics", {"diagnostics": {"kriging_variance": True, "extra": 7}}), + } + + async def call(self, entry_point: str, inputs: dict[str, Any]) -> None: + if entry_point == "run": + await self.client.geostatistics.kriging.run(**inputs) + else: + await self.client.arun("geostatistics", "kriging", inputs) + + async def test_an_extra_field_in_a_folded_input_follows_its_objects_rule(self) -> None: + for name, (node, overrides) in self.folded_inputs().items(): + for rule_name, (rule, outcome) in self.RULES.items(): + for entry_point in ("run", "arun"): + with self.subTest(input=name, rule=rule_name, entry_point=entry_point): + catalogue = mock.AsyncMock(return_value=published_catalogue(node, rule)) + with mock.patch.object(DiscoveryClient, "list_tasks", catalogue): + with _capture_submit("evo.compute.engine") as submit: + if outcome == "rejected": + with self.assertRaises(ParameterValidationError) as caught: + await self.call(entry_point, self.inputs(**overrides)) + else: + logs = ( + self.assertLogs("compute.validation", level="WARNING") + if outcome == "warned" + else self.assertNoLogs("compute.validation", level="WARNING") + ) + with logs as captured, self.assertRaises(_SubmitCaptured): + await self.call(entry_point, self.inputs(**overrides)) + if outcome == "rejected": + self.assertTrue( + any(node in error and "extra" in error for error in caught.exception.errors), + caught.exception.errors, + ) + submit.assert_not_awaited() + continue + if outcome == "warned": + self.assertTrue(any(f"at {node}: 'extra'" in line for line in captured.output)) + wire = submit.await_args.kwargs["parameters"] + for part in node.split("."): + wire = wire[part] + self.assertEqual(7, wire["extra"]) + + async def test_an_extra_field_deep_inside_a_filter_is_kept_where_it_was_written(self) -> None: + """The captured catalogue closes every filter node, so the field is refused at its own location.""" + cases = { + "source.filter.where": {**self.CONDITION, "extra": 7}, + "source.filter.where.filters.1": { + "type": "all_of", + "filters": [self.CONDITION, {**self.CONDITION, "extra": 7}], + }, + } + for location, where in cases.items(): + with self.subTest(location=location): + catalogue = mock.AsyncMock(return_value=published_catalogue()) + with mock.patch.object(DiscoveryClient, "list_tasks", catalogue): + with _capture_submit("evo.compute.engine") as submit: + with self.assertRaises(ParameterValidationError) as caught: + await self.client.arun( + "geostatistics", "kriging", self.inputs(source_filter={"where": where}) + ) + self.assertEqual([f"{location}: unexpected parameter(s): 'extra'"], caught.exception.errors) + submit.assert_not_awaited() + + async def test_fields_the_typed_models_read_are_not_taken_for_extras(self) -> None: + """``ranges`` is renamed on the way out and ``False`` turns a diagnostic off; neither is unknown.""" + neighborhood = { + "ellipsoid": { + "ranges": {"major": 200.0, "semi_major": 150.0, "minor": 100.0}, + "rotation": {"dip_azimuth": 45.0, "dip": 10.0, "pitch": 5.0}, + }, + "max_samples": 20, + } + diagnostics = {"kriging_variance": True, "kriging_mean": False} + catalogue = mock.AsyncMock(return_value=published_catalogue()) + with ( + mock.patch.object(DiscoveryClient, "list_tasks", catalogue), + _capture_submit("evo.compute.engine") as submit, + ): + with self.assertRaises(_SubmitCaptured): + await self.client.arun( + "geostatistics", "kriging", self.inputs(neighborhood=neighborhood, diagnostics=diagnostics) + ) + payload = submit.await_args.kwargs["parameters"] + self.assertEqual({"ellipsoid_ranges", "rotation"}, set(payload["neighborhood"]["ellipsoid"])) + self.assertEqual({"kriging_variance"}, set(payload["target"]["diagnostics"])) + + +# --------------------------------------------------------------------------- # +# Payload fidelity +# --------------------------------------------------------------------------- # + + +class TestPayloadFidelity(OverrideTestCase): + """One set of inputs, three paths, one payload.""" + + async def assertThreeWayParity(self, **inputs: Any) -> None: + """Require the runner, the override and the generic engine to submit the same payload.""" + runner_payload = await self.runner_payload(KrigingRunner, KrigingRunner.params_type(**inputs)) + candidates = { + "override": await self.override_payload(**inputs), + "engine": await self.engine_payload(KrigingRunner, **inputs), + } + for label, payload in candidates.items(): + if differences := payload_differences(runner_payload, payload): + self.fail(f"{label} payload diverges from the runner payload:\n " + "\n ".join(differences)) + + async def test_the_everyday_call(self) -> None: + """Source, target, variogram and a search, with the method left at ordinary.""" + await self.assertThreeWayParity(**self.inputs()) + + async def test_a_tagged_method_object(self) -> None: + """Simple kriging carries a mean, and the tag that says which branch of the union it is.""" + await self.assertThreeWayParity(**self.inputs(kriging_method=KrigingMethod.simple(mean=12.5))) + + async def test_a_typed_attribute_target_becomes_a_create_operation(self) -> None: + """The override takes the typed inputs the runner takes, and expands them the same way.""" + await self.assertThreeWayParity(**self.inputs(target=_pending_attribute("kriged_grade", TARGET_URL))) + + async def test_a_folded_filter_lands_where_the_wire_carries_it(self) -> None: + """``source_filter`` is a model input the serializer writes into ``source``. + + It is an argument the generic engine does not have -- the catalogue never advertises + it -- which is one of the things a hand-written surface is for. + """ + source_filter = Filter(where=FilterCondition(attribute=GRADE_ATTRIBUTE, operator="greater_than", threshold=0.5)) + payload = await self.override_payload(**self.inputs(source_filter=source_filter)) + self.assertNotIn("source_filter", payload) + self.assertEqual(source_filter.model_dump(), payload["source"]["filter"]) + + async def test_an_unset_optional_is_not_sent(self) -> None: + """The override sends what it was given, so the platform's own defaults still apply.""" + payload = await self.override_payload(**self.inputs()) + self.assertNotIn("block_discretization", payload) + + async def test_preview_requires_explicit_opt_in(self) -> None: + catalogue = kriging_catalogue(feature_flag="geostatistics-preview") + with mock.patch.object(DiscoveryClient, "list_tasks", mock.AsyncMock(return_value=catalogue)): + with _capture_submit("evo.compute.engine") as submit: + with self.assertRaises(_SubmitCaptured): + await self.client.geostatistics.kriging.run(**self.inputs()) + self.assertFalse(submit.await_args.kwargs["preview"]) + + with self.assertRaises(_SubmitCaptured): + await self.client.geostatistics.kriging.run(**self.inputs(), preview=True) + self.assertTrue(submit.await_args.kwargs["preview"]) + self.assertNotIn("preview", submit.await_args.kwargs["parameters"]) diff --git a/packages/evo-compute/tests/test_payload_parity.py b/packages/evo-compute/tests/test_payload_parity.py index 8d7d95a78..9ba85701d 100644 --- a/packages/evo-compute/tests/test_payload_parity.py +++ b/packages/evo-compute/tests/test_payload_parity.py @@ -320,7 +320,7 @@ async def engine_payload(self, runner_cls, **parameters: Any) -> dict[str, Any]: ) with catalogue, _capture_submit("evo.compute.engine") as submit: with self.assertRaises(_SubmitCaptured): - await getattr(self.client.geostatistics, runner_cls.task.replace("-", "_")).run(**parameters) + await self.client._run_generic("geostatistics", runner_cls.task, parameters) return submit.await_args.kwargs["parameters"] async def payload_difference(self, runner_cls, **inputs: Any) -> list[str]: diff --git a/packages/evo-compute/tests/test_resolution.py b/packages/evo-compute/tests/test_resolution.py index 2913a715a..dc41fb5a5 100644 --- a/packages/evo-compute/tests/test_resolution.py +++ b/packages/evo-compute/tests/test_resolution.py @@ -473,9 +473,9 @@ async def test_an_explicit_none_survives_resolution(self) -> None: resolved = await self.resolve(source={"object": POINTSET_URL, "attribute": "grade"}, report=None) self.assertIsNone(resolved["report"]) - async def test_an_undeclared_parameter_is_left_for_the_platform_to_judge(self) -> None: - resolved = await self.resolver.resolve(self.spec, {"mystery": {"a": 1}}) - self.assertEqual({"a": 1}, resolved["mystery"]) + async def test_a_closed_object_rejects_undeclared_parameters(self) -> None: + with self.assertRaisesRegex(ParameterValidationError, "mystery"): + await self.resolver.resolve(self.spec, {"mystery": {"a": 1}}) async def test_a_typed_model_in_an_optional_slot_is_dumped(self) -> None: """An ``X | None`` slot has no ``properties`` of its own, but a model there is still ours.""" diff --git a/packages/evo-compute/tests/test_stubgen.py b/packages/evo-compute/tests/test_stubgen.py index ab4a9d8a2..b56d20f18 100644 --- a/packages/evo-compute/tests/test_stubgen.py +++ b/packages/evo-compute/tests/test_stubgen.py @@ -37,6 +37,7 @@ from evo.compute import ComputeClient, ParameterValidationError, _stubgen from evo.compute.endpoints.models import TaskResource from evo.compute.engine import _signature_from_schema, _wire_names +from evo.compute.overrides import overridden_tasks from evo.compute.validation import validate_parameters _CHECKS_DIR = _stubgen.DEFAULT_SNAPSHOT_DIR.parent / "checks" @@ -146,11 +147,27 @@ def test_every_snapshotted_task_is_reachable(self) -> None: def test_stub_signatures_match_the_runtime_signatures(self) -> None: """What the stub advertises is what :func:`_signature_from_schema` will bind.""" for task in _stubgen.load_snapshot(_stubgen.DEFAULT_SNAPSHOT_DIR): + if (task.topic, task.name.replace("-", "_")) in overridden_tasks(): + continue with self.subTest(task=f"{task.topic}.{task.name}"): stub = _stubgen._TaskRenderer(task).render() declared = [parameter.split(":")[0] for parameter in stub.run_parameters] self.assertEqual(list(_signature_from_schema(task).parameters), declared) + def test_an_overridden_task_imports_its_runner(self) -> None: + source = _stubgen.generate_stub() + for topic, task in overridden_tasks(): + with self.subTest(topic=topic, task=task): + module, runner = _stubgen._override_runner(topic, task) + alias = f"_{_stubgen._camel(topic)}{_stubgen._camel(task)}" + self.assertIn(f"from {module} import {runner} as {alias}", source) + self.assertNotIn(f"class {alias}:", source) + + def test_overrides_do_not_depend_on_the_snapshot(self) -> None: + source = _stubgen._render([_spec()], _stubgen.DEFAULT_SNAPSHOT_DIR) + for topic, task in overridden_tasks(): + self.assertIn(f" {task}: _{_stubgen._camel(topic)}{_stubgen._camel(task)}", source) + def test_the_client_signatures_are_not_hand_maintained_into_drift(self) -> None: """The task surface is generated, but the clients' own methods are written out. @@ -466,10 +483,10 @@ def test_sibling_constants_fold_into_one_literal(self) -> None: class TestTaskSurface(unittest.TestCase): - def test_preview_defaults_to_the_feature_flag(self) -> None: + def test_preview_is_always_an_explicit_opt_in(self) -> None: self.assertIn("preview: bool = False", _stubgen._TaskRenderer(_spec()).render().run_parameters) gated = _spec(feature_flag="widgets-preview") - self.assertIn("preview: bool = True", _stubgen._TaskRenderer(gated).render().run_parameters) + self.assertIn("preview: bool = False", _stubgen._TaskRenderer(gated).render().run_parameters) def test_required_parameters_come_first_and_have_no_default(self) -> None: spec = _spec( diff --git a/packages/evo-compute/tests/test_validation.py b/packages/evo-compute/tests/test_validation.py index 941360d61..f661201cb 100644 --- a/packages/evo-compute/tests/test_validation.py +++ b/packages/evo-compute/tests/test_validation.py @@ -23,7 +23,7 @@ from data import load_test_data from evo.compute import ComputeClient, ParameterValidationError, TaskResource -from evo.compute.validation import validate_parameters +from evo.compute.validation import _warn_unknown_fields, validate_parameters def make_spec(parameters: dict[str, Any], *, topic: str = "demo", name: str = "widget") -> TaskResource: @@ -31,6 +31,71 @@ def make_spec(parameters: dict[str, Any], *, topic: str = "demo", name: str = "w return TaskResource.model_validate({"topic": topic, "name": name, "parameters": parameters}) +class TestAdditionalPropertiesPolicy(unittest.TestCase): + def test_closed_objects_reject_unknown_fields(self) -> None: + with self.assertRaisesRegex(ParameterValidationError, "new_option"): + _warn_unknown_fields( + {"properties": {}, "additionalProperties": False}, + {"new_option": True}, + ["config"], + "demo.widget", + ) + + def test_additional_property_values_are_validated(self) -> None: + with self.assertRaisesRegex(ParameterValidationError, "new_count"): + _warn_unknown_fields( + {"additionalProperties": {"type": "integer", "minimum": 1}}, + {"new_count": "invalid"}, + ["counts"], + "demo.widget", + ) + + def test_explicitly_open_objects_do_not_warn(self) -> None: + with self.assertNoLogs("compute.validation", level="WARNING"): + _warn_unknown_fields( + {"properties": {}, "additionalProperties": True}, + {"new_option": True}, + [], + "demo.widget", + ) + + def test_unspecified_policy_warns_without_exposing_values(self) -> None: + with self.assertLogs("compute.validation", level="WARNING") as logs: + _warn_unknown_fields({"properties": {}}, {"new_option": "private value"}, [], "demo.widget") + self.assertIn("to be submitted as is", logs.output[0]) + self.assertNotIn("private value", logs.output[0]) + self.assertNotIn("wire format", logs.output[0]) + + def test_additional_value_schemas_can_reference_root_definitions(self) -> None: + node = {"additionalProperties": {"$ref": "#/$defs/Count"}} + root = {"$defs": {"Count": {"type": "integer", "minimum": 1}}} + _warn_unknown_fields(node, {"new_count": 2}, ["counts"], "demo.widget", root=root) + with self.assertRaisesRegex(ParameterValidationError, "counts.new_count"): + _warn_unknown_fields(node, {"new_count": 0}, ["counts"], "demo.widget", root=root) + + def test_an_empty_additional_schema_accepts_values_without_warning(self) -> None: + with self.assertNoLogs("compute.validation", level="WARNING"): + _warn_unknown_fields( + {"additionalProperties": {}}, + {"new_option": {"nested": [None, True]}}, + [], + "demo.widget", + ) + + def test_pattern_properties_are_not_reported_as_extra_fields(self) -> None: + with self.assertNoLogs("compute.validation", level="WARNING"): + _warn_unknown_fields( + {"patternProperties": {"^count_": {"type": "integer"}}, "additionalProperties": False}, + {"count_samples": 2}, + [], + "demo.widget", + ) + + def test_unspecified_policy_warns_for_free_form_objects_too(self) -> None: + with self.assertLogs("compute.validation", level="WARNING"): + _warn_unknown_fields({"type": "object"}, {"new_option": True}, [], "demo.widget") + + class TestValidateParameters(unittest.TestCase): """Unit tests for the schema-driven parameter validator (no network).""" @@ -47,8 +112,8 @@ def test_shallow_rejects_missing_required(self) -> None: self.assertIn("source", str(ctx.exception)) self.assertEqual(["missing required parameter 'source'"], ctx.exception.errors) - def test_shallow_rejects_required_set_to_none(self) -> None: - """A required field set to None (when its schema is non-nullable) is caught.""" + def test_shallow_rejects_required_resolved_to_none(self) -> None: + """A required field dropped to None (so it would be omitted on the wire) is caught.""" spec = make_spec({"type": "object", "properties": {"source": {"type": "object"}}, "required": ["source"]}) with self.assertRaises(ParameterValidationError): validate_parameters(spec, {"source": None}) @@ -179,6 +244,92 @@ def test_deep_accepts_anyof_union(self) -> None: validate_parameters(spec, {"shape": "circle"}, deep=True) validate_parameters(spec, {"shape": 3}, deep=True) + def test_closed_objects_reject_new_fields_without_weakening_known_constraints(self) -> None: + spec = make_spec( + { + "type": "object", + "properties": {"config": {"$ref": "#/$defs/Config"}}, + "required": ["config"], + "additionalProperties": False, + "$defs": { + "Config": { + "type": "object", + "properties": {"count": {"type": "integer", "minimum": 1}}, + "required": ["count"], + "additionalProperties": False, + } + }, + } + ) + parameters = {"config": {"count": 2}} + validate_parameters(spec, parameters, deep=True) + self.assertEqual({"config": {"count": 2}}, parameters) + + for invalid in ({}, {"count": "2"}, {"count": 0}, {"count": 2, "new_option": None}): + with self.subTest(invalid=invalid), self.assertRaises(ParameterValidationError): + validate_parameters(spec, {"config": invalid}, deep=True) + with self.assertRaisesRegex(ParameterValidationError, "new_parameter"): + validate_parameters(spec, {**parameters, "new_parameter": False}, deep=True) + + def test_closed_discriminated_union_branches_reject_new_fields(self) -> None: + spec = make_spec( + { + "type": "object", + "properties": { + "method": { + "discriminator": {"propertyName": "kind"}, + "oneOf": [ + { + "type": "object", + "properties": {"kind": {"const": "ordinary"}}, + "required": ["kind"], + "additionalProperties": False, + }, + { + "type": "object", + "properties": {"kind": {"const": "simple"}, "mean": {"type": "number"}}, + "required": ["kind", "mean"], + "additionalProperties": False, + }, + ], + } + }, + } + ) + validate_parameters(spec, {"method": {"kind": "simple", "mean": 2.0}}, deep=True) + with self.assertRaisesRegex(ParameterValidationError, "new_option"): + validate_parameters(spec, {"method": {"kind": "simple", "mean": 2.0, "new_option": True}}, deep=True) + with self.assertRaisesRegex(ParameterValidationError, r"method\.mean"): + validate_parameters(spec, {"method": {"kind": "simple", "mean": "invalid"}}, deep=True) + + def test_typed_maps_and_schema_literals_keep_their_constraints(self) -> None: + spec = make_spec( + { + "type": "object", + "properties": { + "counts": {"type": "object", "additionalProperties": {"type": "integer", "minimum": 1}}, + "literal": {"const": {"additionalProperties": False, "unevaluatedProperties": False}}, + }, + "unevaluatedProperties": False, + } + ) + original = json.dumps(spec.parameters, sort_keys=True) + literal = {"additionalProperties": False, "unevaluatedProperties": False} + validate_parameters(spec, {"counts": {"new_counter": 2}, "literal": literal}, deep=True) + self.assertEqual(original, json.dumps(spec.parameters, sort_keys=True)) + with self.assertRaisesRegex(ParameterValidationError, "new_field"): + validate_parameters(spec, {"new_field": None}, deep=True) + for invalid in ("2", 0): + with ( + self.subTest(invalid=invalid), + self.assertRaisesRegex(ParameterValidationError, r"counts\.new_counter"), + ): + validate_parameters(spec, {"counts": {"new_counter": invalid}}, deep=True) + with self.assertRaisesRegex(ParameterValidationError, "literal"): + validate_parameters( + spec, {"literal": {"additionalProperties": True, "unevaluatedProperties": True}}, deep=True + ) + # -- reference-leaf relaxation ----------------------------------------- # def test_deep_relaxes_a_reference_leaf_declared_as_its_resolved_form(self) -> None: @@ -193,7 +344,7 @@ def test_deep_relaxes_a_reference_leaf_declared_as_its_resolved_form(self) -> No validate_parameters(spec, {"source": "https://example/objects/abc"}, deep=True) # does not raise def test_deep_holds_a_scalar_reference_to_its_declared_type(self) -> None: - """A reference transmitted as a string is still checked against its scalar type.""" + """A reference transmitted as a string is still checked -- that is what the stub promises.""" spec = make_spec( { "type": "object", @@ -233,7 +384,12 @@ def test_deep_keeps_the_transmitted_branch_of_a_mixed_declared_type(self) -> Non validate_parameters(spec, {"source": 12345}, deep=True) def test_deep_relaxes_a_nullable_resolved_form(self) -> None: - """``null`` says the leaf may be omitted, not that a reference travels as one.""" + """``null`` says the leaf may be omitted, not that a reference travels as one. + + Counting it as a transmitted form would keep ``["object", "null"]`` declared, and + resolution has already turned the reference into a URL by the time deep validation + sees it -- so every non-null call would be rejected. + """ spec = make_spec( { "type": "object",