From f6e5b757a8950f7c040c7828f8ee24a991401e6c Mon Sep 17 00:00:00 2001 From: olav Date: Tue, 8 Sep 2026 23:32:41 +0200 Subject: [PATCH] feat(events)!: advancedFilter becomes a filter expression BREAKING CHANGE: `EventFilterForm::set_advanced_filter` takes a string instead of an `AdvancedEventFilter`, and the `Filter`, `AdvancedEventFilter` and `RelatedResourceFilter` types are gone. The api replaced the nested and/or/not JSON with a boolean expression it parses itself: form.set_advanced_filter("type NOT LIKE 'pump' AND (subType = 'water' OR subType = 'gas')"); The tree was worth less than it looked. Of the nine `Filter` variants, three -- Range, ContainsAny, ContainsAll -- had no server-side binding at all and returned a 500, and `IsSet` reached a code path that could not read the property it was given. Nothing outside `src/filters.rs` constructed one: `Filter` was never re-exported at the crate root, `RelatedResourceFilter` had no public constructor, and the Python bindings never exposed any of it. So the blast radius is small for something this loud. The field name and its serde attributes are unchanged, so the wire key is still `advancedFilter` and an unset filter still omits it rather than sending `null` -- which matters, because the api rejects unknown and null-typed keys. `advanced_filter_serializes_as_camel_case` stays as the guard for that and now also pins the value shape. Python gains the parameter in the same release rather than being left without advanced filtering: `advanced_filter=` on both `filter()` methods, through the shared `event_filter_form` builder, with the two stubs updated. The expression is validated by the api, not here. An invalid one comes back as a 400 carrying an offset and, where the fix is unambiguous, the corrected expression -- so a client can underline the mistake or offer the repair without knowing the grammar. Version 0.4.0: removing public types is breaking, and under Cargo's 0.x rules that is a minor bump. All three manifests move together, as the versions CI job requires. Signed-off-by: olav --- AGENTS.md | 2 +- Cargo.toml | 2 +- datahub_python_bindings/Cargo.toml | 2 +- datahub_python_bindings/pyproject.toml | 2 +- .../intellistream_datahub_sdk/__init__.pyi | 2 + .../src/events/async_service.rs | 6 +- datahub_python_bindings/src/events/mod.rs | 8 + .../src/events/sync_service.rs | 6 +- docs/api-coverage.md | 8 +- src/filters.rs | 330 ++---------------- 10 files changed, 55 insertions(+), 313 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ddf6adf..b6281ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -185,7 +185,7 @@ Names the refactors removed — `externalIdPrefix`, `metadataKey`/`metadataValue Related resources are one field, not two: both `Event` and `EventFilter` carry `relatedResources`, an array of the backend's `IdCollection` (`[{"id": "34"}, {"externalId": "sensor_abc"}]`, modelled by `IdAndExtId`). An entry may name a resource by id, external id, or both; the backend resolves the missing side and returns both. `dataSetId` on the filter uses the same shape. There are no aliases for the retired flat `relatedResourceIds` / `relatedResourceExternalIds` arrays. -`EventFilterForm` + `AdvancedEventFilter` remain the richer style for events; some advanced-filter endpoints are not yet wired up server-side and are tested only via serde round-trips. +`EventFilterForm` additionally carries `advancedFilter`, a **string** holding a boolean expression in the api's PostgreSQL-flavoured filter language — `type NOT LIKE 'pump' AND (subType = 'water' OR subType = 'gas')`. It replaced a nested and/or/not `Filter` tree in 0.4.0. Three of that tree's nine variants (`range`, `containsAny`, `containsAll`) never had a server-side binding and returned 500, which is part of why it went. The expression is parsed and validated by the api, not here, so an invalid one comes back as a 400 carrying an offset and often a corrected expression. #### Sorting and paging diff --git a/Cargo.toml b/Cargo.toml index 34e6569..932774f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "intellistream-datahub-sdk" -version = "0.3.0" +version = "0.4.0" edition = "2021" rust-version = "1.85" description = "Async Rust SDK for the IntelliStream DataHub REST API: time series and datapoints, events, resources and relationship edges, files, datasets, labels and subscriptions." diff --git a/datahub_python_bindings/Cargo.toml b/datahub_python_bindings/Cargo.toml index e0efec4..6567061 100644 --- a/datahub_python_bindings/Cargo.toml +++ b/datahub_python_bindings/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datahub_python_bindings" -version = "0.3.0" +version = "0.4.0" edition = "2024" rust-version = "1.85" description = "PyO3 bindings for the IntelliStream DataHub SDK, published to PyPI as intellistream-datahub-sdk." diff --git a/datahub_python_bindings/pyproject.toml b/datahub_python_bindings/pyproject.toml index 8a0f72a..e678e1a 100644 --- a/datahub_python_bindings/pyproject.toml +++ b/datahub_python_bindings/pyproject.toml @@ -11,7 +11,7 @@ exclude = ["**/__pycache__/**"] [project] name = "intellistream-datahub-sdk" -version = "0.3.0" +version = "0.4.0" description = "Python bindings for the IntelliStream DataHub SDK: time series and datapoints, events, resources and relationship edges, files, datasets, labels and subscriptions." readme = "README.md" authors = [{ name = "IntelliStream AS", email = "support@intellistream.ai" }] diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index accc707..9dc37f6 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -899,6 +899,7 @@ class EventsServiceSync: sort_by: SortBy | None = None, sort_order: str | None = None, cursor: str | None = None, + advanced_filter: str | None = None, ) -> Page: """Pass either ``filter=`` or the individual criteria keywords; passing both is a ``TypeError``. Paging is always given here rather than on the filter, so one filter can be @@ -954,6 +955,7 @@ class EventsServiceAsync: sort_by: SortBy | None = None, sort_order: str | None = None, cursor: str | None = None, + advanced_filter: str | None = None, ) -> Page: """Pass either ``filter=`` or the individual criteria keywords; passing both is a ``TypeError``. Paging is always given here rather than on the filter, so one filter can be diff --git a/datahub_python_bindings/src/events/async_service.rs b/datahub_python_bindings/src/events/async_service.rs index 86dc724..5d7328d 100644 --- a/datahub_python_bindings/src/events/async_service.rs +++ b/datahub_python_bindings/src/events/async_service.rs @@ -124,7 +124,8 @@ impl PyEventsServiceAsync { #[pyo3(signature = (filter=None, external_id=None, source=None, r#type=None, sub_type=None, status=None, data_set_id=None, event_time=None, metadata=None, related_resources=None, created_time=None, last_updated_time=None, - limit=None, sort_by=None, sort_order=None, cursor=None))] + limit=None, sort_by=None, sort_order=None, cursor=None, + advanced_filter=None))] #[allow(clippy::too_many_arguments)] fn filter<'py>( &self, @@ -145,11 +146,12 @@ impl PyEventsServiceAsync { sort_by: Option, sort_order: Option, cursor: Option, + advanced_filter: Option, ) -> PyResult> { let form = crate::events::event_filter_form( filter, external_id, source, r#type, sub_type, status, data_set_id, event_time, metadata, related_resources, created_time, last_updated_time, limit, sort_by, - sort_order, cursor, + sort_order, cursor, advanced_filter, )?; let service = self.api_service.clone(); diff --git a/datahub_python_bindings/src/events/mod.rs b/datahub_python_bindings/src/events/mod.rs index 263ffa7..81863f2 100644 --- a/datahub_python_bindings/src/events/mod.rs +++ b/datahub_python_bindings/src/events/mod.rs @@ -86,6 +86,7 @@ pub fn event_filter_form( sort_by: Option, sort_order: Option, cursor: Option, + advanced_filter: Option, ) -> PyResult { let any_keyword = external_id.is_some() || source.is_some() @@ -131,6 +132,13 @@ pub fn event_filter_form( if let Some(cursor) = cursor { form.set_cursor(cursor); } + // A boolean expression in the api's filter language, e.g. + // type NOT LIKE 'pump' AND (subType = 'water' OR subType = 'gas') + // Parsed and validated by the api, so an invalid one comes back as a 400 that carries an + // offset and usually a corrected expression. + if let Some(advanced_filter) = advanced_filter { + form.set_advanced_filter(advanced_filter); + } Ok(form.build()) } diff --git a/datahub_python_bindings/src/events/sync_service.rs b/datahub_python_bindings/src/events/sync_service.rs index 7085774..b851051 100644 --- a/datahub_python_bindings/src/events/sync_service.rs +++ b/datahub_python_bindings/src/events/sync_service.rs @@ -108,7 +108,8 @@ impl PyEventsServiceSync { #[pyo3(signature = (filter=None, external_id=None, source=None, r#type=None, sub_type=None, status=None, data_set_id=None, event_time=None, metadata=None, related_resources=None, created_time=None, last_updated_time=None, - limit=None, sort_by=None, sort_order=None, cursor=None))] + limit=None, sort_by=None, sort_order=None, cursor=None, + advanced_filter=None))] #[allow(clippy::too_many_arguments)] fn filter<'py>( &self, @@ -129,11 +130,12 @@ impl PyEventsServiceSync { sort_by: Option, sort_order: Option, cursor: Option, + advanced_filter: Option, ) -> PyResult { let form = crate::events::event_filter_form( filter, external_id, source, r#type, sub_type, status, data_set_id, event_time, metadata, related_resources, created_time, last_updated_time, limit, sort_by, - sort_order, cursor, + sort_order, cursor, advanced_filter, )?; let service = self.api_service.clone(); diff --git a/docs/api-coverage.md b/docs/api-coverage.md index 7e35be4..e5875dc 100644 --- a/docs/api-coverage.md +++ b/docs/api-coverage.md @@ -90,8 +90,9 @@ vocabularies). All take a `limit` param, default 1000, capped at 10000. Covered: `byids`, `search`, `fetch-related`, `create`, `update`, `delete`. Missing: `GET /resources/{id}` (single fetch by numeric id), `POST /resources/filter` (structured -filtering — note the SDK ships `AdvancedFilter` machinery in `src/filters.rs` with no resource -endpoint wired to it), `POST /resources/fetch-nearest`. +filtering), `POST /resources/fetch-nearest`. Note that events' `advancedFilter` is now a plain +expression string and carries no reusable type, so a resource equivalent would start from the +api's filter language rather than from anything in `src/filters.rs`. ### `/timeseries` — 11 of 12 @@ -149,7 +150,8 @@ knowing, found while testing against a live backend: 3. ~~`GET /files/download/{id}`~~ — **done**, along with the rest of `/files`. 4. **`/edges` service** — needed for any relationship management that isn't a resource-create side effect, and for discovering relationship types. -5. **`/resources/filter`** — the `AdvancedFilter` types already exist; only the call is missing. +5. **`/resources/filter`** — needs both the call and whatever filter shape the api settles on for + resources; the events filter language is a string, with no Rust-side types to reuse. 6. **`/policies` and `/governance`** — newer surfaces; wire up when the platform needs them. 7. Lower value: events vocabulary enumeration, `/stats`, `/tenant/features`, `recommend-value-type`, the live datapoint WebSocket. diff --git a/src/filters.rs b/src/filters.rs index 48bbb34..1602720 100644 --- a/src/filters.rs +++ b/src/filters.rs @@ -352,8 +352,24 @@ pub struct EventFilterForm { cursor: Option, #[serde(skip_serializing_if = "Option::is_none")] sort: Option, + /// A boolean filter expression, sent as `advancedFilter`. + /// + /// Replaces the nested and/or/not `Filter` tree this used to hold. The api parses the string + /// into a tree of its own and renders it as a parameterised query, so what travels here is + /// what a person would type into a WHERE clause: + /// + /// ```text + /// type NOT LIKE 'pump' AND (subType = 'water' OR subType = 'gas') + /// ``` + /// + /// The dialect is PostgreSQL-flavoured: PostgreSQL function names, `::` casts, ILIKE, and + /// `<>` beside `!=`. Event metadata values are text, so comparing one as anything else needs + /// a converter -- `to_int`, `to_number`, `to_bool`, `to_date`, `to_timestamp`. + /// + /// Still `skip_serializing_if`: the api rejects unknown and null-typed keys, so an unset + /// filter has to omit the key rather than send `null`. #[serde(skip_serializing_if = "Option::is_none")] - advanced_filter: Option, + advanced_filter: Option, } impl EventFilterForm { @@ -422,312 +438,21 @@ impl EventFilterForm { self.limit = limit; self } - pub fn set_advanced_filter(&mut self, filter: AdvancedEventFilter) -> &mut Self { - self.advanced_filter = Some(filter); - self - } - pub fn build(&self) -> Self { - self.clone() - } -} - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct AdvancedEventFilter { - filter: Option, - #[serde(skip_serializing_if = "Option::is_none")] - related_resource_filter: Option, -} - -impl AdvancedEventFilter { - pub fn new() -> Self { - Self { - filter: None, // filter that the returned event must satisfy - related_resource_filter: None, // idea was to filter so that if - } - } - pub fn set_filter(&mut self, filter: &Filter) -> &mut Self { - self.filter = Some(filter.clone()); - self - } - pub fn set_related_resource_filter(&mut self, filter: &RelatedResourceFilter) -> &mut Self { - self.related_resource_filter = Some(filter.clone()); + /// Set the boolean filter expression. See [`EventFilterForm::advanced_filter`]. + pub fn set_advanced_filter(&mut self, expression: impl Into) -> &mut Self { + self.advanced_filter = Some(expression.into()); self } pub fn build(&self) -> Self { self.clone() } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct RelatedResourceFilter { - // Idea is that you can filter on events that have spesific neighbors. - // it will apply the filter on the neighbors if the relation type matches - // can see this being useful, but we should be careful as - // it can be very complicated and potentialy computatuinaly expensive - // relatedResource probably dont need to check their neighbors aswell - // - // example case: we have a pump that will be worked on with - // a work permit that will be a related resource? - // assume pump1 exists and will produce a timeseries RPM_pump1 with associated events - // like "Pump1 RPM below threshold" - // when the work permit is activated pump1 will be updated indicating it has an active work permit on it - // the produced event will have a related resource /source field indicating its source ie pump1 - // using the related resource filter we can filter on events that dont have an active work permit? - // can probalby do a lot more complex stuff aswell. - filter: Filter, - relation_types: Option>, -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -/// -/// Constructs an arbitrary boolean statement for advanced filtering. -/// -/// # Variants -/// -/// - And(Vec): Combines 2 or more filters on logical and. -/// -/// - Or(Vec): Combines 2 or more filters on logical or. -/// -/// - Not(Box): Negates the result of the child -/// -/// - Equals: Porperty value must match the given value exactly. -/// - property: The property name to be evaluated. -/// - value: The value to match against the property. -/// -/// - In: property matches any of a list of values. -/// - property: The property name to be evaluated. -/// - values: The list of values to check for inclusion. -/// -/// - Range: check if property is between min and max values -/// - property: The property name to be evaluated. -/// - min: The optional lower bound of the range (inclusive or exclusive support could be added in the future). -/// - max: The optional upper bound of the range (inclusive or exclusive support could be added in the future). -/// -/// - IsSet: check listed properties are not None -/// - property: A vector of property names to check. -/// -/// - ContainsAny: property (list-type) contains at least 1 element in any_of -/// - property: The property name to be evaluated. -/// - any_of: The list of values to check for presence. -/// -/// - ContainsAll: property (list-type) contains all elements in all_of -/// - property: The property name to be evaluated. -/// - all_of: The list of values that must all be present. -/// -/// # Usage -/// -/// This filter can be used to construct an arbitrary boolean statement: -/// -/// ```rust -/// use serde_json; -/// use intellistream_datahub_sdk::{filters::Filter}; -/// use serde_json::json; -/// let filter = Filter::And(vec![ -/// Filter::Equals { -/// property: "status".to_string(), -/// value: "active".to_string() -/// }, -/// Filter::Not(Box::new(Filter::In { -/// property: "category".to_string(), -/// values: vec!["restricted".to_string(), "archived".to_string()] -/// })), -/// ]); -/// let serialized = serde_json::to_string(&filter).unwrap(); -/// println!("{}", serialized); -/// let deserialized: Filter = serde_json::from_str(&serialized).unwrap(); -/// assert_eq!(filter, deserialized); -///``` -/// // -#[serde(rename_all = "camelCase")] -pub enum Filter { - // filters are constructed as a tree like structure using the standard boolean operators AND, OR, NOT, - And(Vec), - Or(Vec), - Not(Box), - // these filters assume the property is a single value - Equals { - // evaluate if the given property is equal to the given value - property: String, - value: String, - }, - In { - // evaluate if any of the given values are equal to the given property - property: String, - values: Vec, - }, - Range { - // evaluate if the given property is between the given min and max values - // in future we could add support for inclusive/exclusive bounds - max: Option, - min: Option, - property: String, - }, - IsSet { - // evaluate if the given property is set (not null) - property: Vec, - }, - // the remaining filters assume a property is a list - #[serde(rename_all = "camelCase")] - ContainsAny { - // check if the property contains any of the given values - any_of: Vec, - property: String, - }, - #[serde(rename_all = "camelCase")] - ContainsAll { - all_of: Vec, - property: String, - }, -} - -impl Filter { - pub fn and(filters: &Vec) -> Self { - Filter::And(filters.clone()) - } - pub fn or(filters: &Vec) -> Self { - Filter::Or(filters.clone()) - } - pub fn not(filter: &Filter) -> Self { - Filter::Not(Box::new(filter.clone())) - } - pub fn eq(property: &str, value: &str) -> Self { - Filter::Equals { - property: property.to_string(), - value: value.to_string(), - } - } - pub fn in_values(property: &str, values: &Vec) -> Self { - Filter::In { - property: property.to_string(), - values: values.clone(), - } - } - pub fn range(property: &str, min: Option, max: Option) -> Self { - Filter::Range { - property: property.to_string(), - min, - max, - } - } - pub fn is_set(property: &Vec) -> Self { - Filter::IsSet { - property: property.clone(), - } - } - pub fn contains_any(property: &str, any_of: &[String]) -> Self { - Filter::ContainsAny { - property: property.to_string(), - any_of: any_of.to_vec(), - } - } - pub fn contains_all(property: &str, all_of: &[String]) -> Self { - Filter::ContainsAll { - property: property.to_string(), - all_of: all_of.to_vec(), - } - } - - pub fn build(&self) -> Self { - self.clone() - } -} #[cfg(test)] mod tests { use super::*; use serde_json::json; - #[test] - fn test_advanced_filter() { - let mut filt = AdvancedEventFilter::new(); - assert_eq!(filt.filter, None); - let leaf_filter1 = Filter::Equals { - property: "test".to_string(), - value: "test".to_string(), - }; - let leaf_filter2 = Filter::contains_all( - "policies", - &vec!["policy1".to_string(), "policy2".to_string()], - ); - let leaf_filter3 = Filter::is_set(&vec!["metdata".to_string(), "type".to_string()]); - let leaf_filter4 = Filter::contains_any( - "policies", - &vec!["policy3".to_string(), "policy4".to_string()], - ); - - let bool_filter1 = Filter::and(&vec![leaf_filter1.clone(), leaf_filter2.clone()]); - let bool_filter2 = Filter::or(&vec![leaf_filter2, leaf_filter4]); - let bool_filter3 = Filter::not(&leaf_filter3); - let expected_json = json!({"filter": {"equals": {"property": "test", "value": "test"}}}); - filt.filter = Some(leaf_filter1.clone()); - assert_eq!( - serde_json::to_string(&filt).unwrap(), - expected_json.to_string() - ); - - let expected_json2 = json!( - {"filter": - {"and":[ - {"equals":{"property": "test", "value": "test"}}, - {"containsAll":{"property":"policies","allOf":["policy1", "policy2"]}} - ] - }}); - assert_eq!( - serde_json::to_string(&AdvancedEventFilter { - filter: Some(bool_filter1.clone()), - related_resource_filter: None - }) - .unwrap(), - expected_json2.to_string() - ); - let expected_json3 = json!({ - "filter": { - "and": [ - { - "or": [ - { - "and": [ - {"equals": {"property": "test", "value": "test"}}, - {"containsAll": {"property": "policies", "allOf": ["policy1", "policy2"]}} - ] - }, - { - "or": [ - {"containsAll": {"property": "policies", "allOf": ["policy1", "policy2"]}}, - {"containsAny": {"property": "policies", "anyOf": ["policy3", "policy4"]}} - ] - } - ] - }, - { - "not": { - "not": { - "isSet": {"property": ["metdata", "type"]} - } - } - } - ] - } - }); - - let filter1_or_2_and_not3 = Filter::and(&vec![ - Filter::or(&vec![bool_filter1.clone(), bool_filter2]), - Filter::not(&bool_filter3), - ]); - assert_eq!( - serde_json::to_string(&AdvancedEventFilter { - filter: Some(filter1_or_2_and_not3), - related_resource_filter: None - }) - .unwrap(), - expected_json3.to_string() - ) - } - - // The backend event filter reads a single `relatedResources: [{id}|{externalId}]` array - // (`Collection`), NOT flat `relatedResourceIds` / `relatedResourceExternalIds` - // arrays. Serializing the flat keys silently disabled related-resource filtering, so lock the - // wire shape here. #[test] fn basic_event_filter_serializes_related_resources_as_id_collection() { let mut filter = EventFilter::default(); @@ -941,16 +666,17 @@ mod tests { } /// `advanced_filter` used to serialize under its snake_case Rust name, which the api does not - /// read — the filter was silently ignored and the query came back unfiltered. + /// know. It is `advancedFilter` on the wire, and now carries an expression rather than a tree. #[test] fn advanced_filter_serializes_as_camel_case() { - let mut filter = EventFilterForm::default(); - filter.set_advanced_filter(AdvancedEventFilter::new()); + let mut form = EventFilterForm::default(); + form.set_advanced_filter("type = 'Alarm' AND subType IS NOT NULL"); + let value = serde_json::to_value(form.build()).unwrap(); - let value = serde_json::to_value(filter.build()).unwrap(); - assert!( - value.get("advancedFilter").is_some(), - "expected advancedFilter, got: {value}" + assert_eq!( + value.get("advancedFilter").and_then(|v| v.as_str()), + Some("type = 'Alarm' AND subType IS NOT NULL"), + "expected the expression under advancedFilter, got: {value}" ); assert!(value.get("advanced_filter").is_none()); }