diff --git a/AGENTS.md b/AGENTS.md index ddf6adf..b6281ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -185,7 +185,7 @@ Names the refactors removed — `externalIdPrefix`, `metadataKey`/`metadataValue Related resources are one field, not two: both `Event` and `EventFilter` carry `relatedResources`, an array of the backend's `IdCollection` (`[{"id": "34"}, {"externalId": "sensor_abc"}]`, modelled by `IdAndExtId`). An entry may name a resource by id, external id, or both; the backend resolves the missing side and returns both. `dataSetId` on the filter uses the same shape. There are no aliases for the retired flat `relatedResourceIds` / `relatedResourceExternalIds` arrays. -`EventFilterForm` + `AdvancedEventFilter` remain the richer style for events; some advanced-filter endpoints are not yet wired up server-side and are tested only via serde round-trips. +`EventFilterForm` additionally carries `advancedFilter`, a **string** holding a boolean expression in the api's PostgreSQL-flavoured filter language — `type NOT LIKE 'pump' AND (subType = 'water' OR subType = 'gas')`. It replaced a nested and/or/not `Filter` tree in 0.4.0. Three of that tree's nine variants (`range`, `containsAny`, `containsAll`) never had a server-side binding and returned 500, which is part of why it went. The expression is parsed and validated by the api, not here, so an invalid one comes back as a 400 carrying an offset and often a corrected expression. #### Sorting and paging diff --git a/Cargo.toml b/Cargo.toml index 34e6569..932774f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "intellistream-datahub-sdk" -version = "0.3.0" +version = "0.4.0" edition = "2021" rust-version = "1.85" description = "Async Rust SDK for the IntelliStream DataHub REST API: time series and datapoints, events, resources and relationship edges, files, datasets, labels and subscriptions." diff --git a/datahub_python_bindings/Cargo.toml b/datahub_python_bindings/Cargo.toml index e0efec4..6567061 100644 --- a/datahub_python_bindings/Cargo.toml +++ b/datahub_python_bindings/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datahub_python_bindings" -version = "0.3.0" +version = "0.4.0" edition = "2024" rust-version = "1.85" description = "PyO3 bindings for the IntelliStream DataHub SDK, published to PyPI as intellistream-datahub-sdk." diff --git a/datahub_python_bindings/pyproject.toml b/datahub_python_bindings/pyproject.toml index 8a0f72a..e678e1a 100644 --- a/datahub_python_bindings/pyproject.toml +++ b/datahub_python_bindings/pyproject.toml @@ -11,7 +11,7 @@ exclude = ["**/__pycache__/**"] [project] name = "intellistream-datahub-sdk" -version = "0.3.0" +version = "0.4.0" description = "Python bindings for the IntelliStream DataHub SDK: time series and datapoints, events, resources and relationship edges, files, datasets, labels and subscriptions." readme = "README.md" authors = [{ name = "IntelliStream AS", email = "support@intellistream.ai" }] diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index accc707..9dc37f6 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -899,6 +899,7 @@ class EventsServiceSync: sort_by: SortBy | None = None, sort_order: str | None = None, cursor: str | None = None, + advanced_filter: str | None = None, ) -> Page: """Pass either ``filter=`` or the individual criteria keywords; passing both is a ``TypeError``. Paging is always given here rather than on the filter, so one filter can be @@ -954,6 +955,7 @@ class EventsServiceAsync: sort_by: SortBy | None = None, sort_order: str | None = None, cursor: str | None = None, + advanced_filter: str | None = None, ) -> Page: """Pass either ``filter=`` or the individual criteria keywords; passing both is a ``TypeError``. Paging is always given here rather than on the filter, so one filter can be diff --git a/datahub_python_bindings/src/events/async_service.rs b/datahub_python_bindings/src/events/async_service.rs index 86dc724..5d7328d 100644 --- a/datahub_python_bindings/src/events/async_service.rs +++ b/datahub_python_bindings/src/events/async_service.rs @@ -124,7 +124,8 @@ impl PyEventsServiceAsync { #[pyo3(signature = (filter=None, external_id=None, source=None, r#type=None, sub_type=None, status=None, data_set_id=None, event_time=None, metadata=None, related_resources=None, created_time=None, last_updated_time=None, - limit=None, sort_by=None, sort_order=None, cursor=None))] + limit=None, sort_by=None, sort_order=None, cursor=None, + advanced_filter=None))] #[allow(clippy::too_many_arguments)] fn filter<'py>( &self, @@ -145,11 +146,12 @@ impl PyEventsServiceAsync { sort_by: Option, sort_order: Option, cursor: Option, + advanced_filter: Option, ) -> PyResult> { let form = crate::events::event_filter_form( filter, external_id, source, r#type, sub_type, status, data_set_id, event_time, metadata, related_resources, created_time, last_updated_time, limit, sort_by, - sort_order, cursor, + sort_order, cursor, advanced_filter, )?; let service = self.api_service.clone(); diff --git a/datahub_python_bindings/src/events/mod.rs b/datahub_python_bindings/src/events/mod.rs index 263ffa7..81863f2 100644 --- a/datahub_python_bindings/src/events/mod.rs +++ b/datahub_python_bindings/src/events/mod.rs @@ -86,6 +86,7 @@ pub fn event_filter_form( sort_by: Option, sort_order: Option, cursor: Option, + advanced_filter: Option, ) -> PyResult { let any_keyword = external_id.is_some() || source.is_some() @@ -131,6 +132,13 @@ pub fn event_filter_form( if let Some(cursor) = cursor { form.set_cursor(cursor); } + // A boolean expression in the api's filter language, e.g. + // type NOT LIKE 'pump' AND (subType = 'water' OR subType = 'gas') + // Parsed and validated by the api, so an invalid one comes back as a 400 that carries an + // offset and usually a corrected expression. + if let Some(advanced_filter) = advanced_filter { + form.set_advanced_filter(advanced_filter); + } Ok(form.build()) } diff --git a/datahub_python_bindings/src/events/sync_service.rs b/datahub_python_bindings/src/events/sync_service.rs index 7085774..b851051 100644 --- a/datahub_python_bindings/src/events/sync_service.rs +++ b/datahub_python_bindings/src/events/sync_service.rs @@ -108,7 +108,8 @@ impl PyEventsServiceSync { #[pyo3(signature = (filter=None, external_id=None, source=None, r#type=None, sub_type=None, status=None, data_set_id=None, event_time=None, metadata=None, related_resources=None, created_time=None, last_updated_time=None, - limit=None, sort_by=None, sort_order=None, cursor=None))] + limit=None, sort_by=None, sort_order=None, cursor=None, + advanced_filter=None))] #[allow(clippy::too_many_arguments)] fn filter<'py>( &self, @@ -129,11 +130,12 @@ impl PyEventsServiceSync { sort_by: Option, sort_order: Option, cursor: Option, + advanced_filter: Option, ) -> PyResult { let form = crate::events::event_filter_form( filter, external_id, source, r#type, sub_type, status, data_set_id, event_time, metadata, related_resources, created_time, last_updated_time, limit, sort_by, - sort_order, cursor, + sort_order, cursor, advanced_filter, )?; let service = self.api_service.clone(); diff --git a/docs/api-coverage.md b/docs/api-coverage.md index 7e35be4..e5875dc 100644 --- a/docs/api-coverage.md +++ b/docs/api-coverage.md @@ -90,8 +90,9 @@ vocabularies). All take a `limit` param, default 1000, capped at 10000. Covered: `byids`, `search`, `fetch-related`, `create`, `update`, `delete`. Missing: `GET /resources/{id}` (single fetch by numeric id), `POST /resources/filter` (structured -filtering — note the SDK ships `AdvancedFilter` machinery in `src/filters.rs` with no resource -endpoint wired to it), `POST /resources/fetch-nearest`. +filtering), `POST /resources/fetch-nearest`. Note that events' `advancedFilter` is now a plain +expression string and carries no reusable type, so a resource equivalent would start from the +api's filter language rather than from anything in `src/filters.rs`. ### `/timeseries` — 11 of 12 @@ -149,7 +150,8 @@ knowing, found while testing against a live backend: 3. ~~`GET /files/download/{id}`~~ — **done**, along with the rest of `/files`. 4. **`/edges` service** — needed for any relationship management that isn't a resource-create side effect, and for discovering relationship types. -5. **`/resources/filter`** — the `AdvancedFilter` types already exist; only the call is missing. +5. **`/resources/filter`** — needs both the call and whatever filter shape the api settles on for + resources; the events filter language is a string, with no Rust-side types to reuse. 6. **`/policies` and `/governance`** — newer surfaces; wire up when the platform needs them. 7. Lower value: events vocabulary enumeration, `/stats`, `/tenant/features`, `recommend-value-type`, the live datapoint WebSocket. diff --git a/src/filters.rs b/src/filters.rs index 48bbb34..1602720 100644 --- a/src/filters.rs +++ b/src/filters.rs @@ -352,8 +352,24 @@ pub struct EventFilterForm { cursor: Option, #[serde(skip_serializing_if = "Option::is_none")] sort: Option, + /// A boolean filter expression, sent as `advancedFilter`. + /// + /// Replaces the nested and/or/not `Filter` tree this used to hold. The api parses the string + /// into a tree of its own and renders it as a parameterised query, so what travels here is + /// what a person would type into a WHERE clause: + /// + /// ```text + /// type NOT LIKE 'pump' AND (subType = 'water' OR subType = 'gas') + /// ``` + /// + /// The dialect is PostgreSQL-flavoured: PostgreSQL function names, `::` casts, ILIKE, and + /// `<>` beside `!=`. Event metadata values are text, so comparing one as anything else needs + /// a converter -- `to_int`, `to_number`, `to_bool`, `to_date`, `to_timestamp`. + /// + /// Still `skip_serializing_if`: the api rejects unknown and null-typed keys, so an unset + /// filter has to omit the key rather than send `null`. #[serde(skip_serializing_if = "Option::is_none")] - advanced_filter: Option, + advanced_filter: Option, } impl EventFilterForm { @@ -422,312 +438,21 @@ impl EventFilterForm { self.limit = limit; self } - pub fn set_advanced_filter(&mut self, filter: AdvancedEventFilter) -> &mut Self { - self.advanced_filter = Some(filter); - self - } - pub fn build(&self) -> Self { - self.clone() - } -} - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct AdvancedEventFilter { - filter: Option, - #[serde(skip_serializing_if = "Option::is_none")] - related_resource_filter: Option, -} - -impl AdvancedEventFilter { - pub fn new() -> Self { - Self { - filter: None, // filter that the returned event must satisfy - related_resource_filter: None, // idea was to filter so that if - } - } - pub fn set_filter(&mut self, filter: &Filter) -> &mut Self { - self.filter = Some(filter.clone()); - self - } - pub fn set_related_resource_filter(&mut self, filter: &RelatedResourceFilter) -> &mut Self { - self.related_resource_filter = Some(filter.clone()); + /// Set the boolean filter expression. See [`EventFilterForm::advanced_filter`]. + pub fn set_advanced_filter(&mut self, expression: impl Into) -> &mut Self { + self.advanced_filter = Some(expression.into()); self } pub fn build(&self) -> Self { self.clone() } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct RelatedResourceFilter { - // Idea is that you can filter on events that have spesific neighbors. - // it will apply the filter on the neighbors if the relation type matches - // can see this being useful, but we should be careful as - // it can be very complicated and potentialy computatuinaly expensive - // relatedResource probably dont need to check their neighbors aswell - // - // example case: we have a pump that will be worked on with - // a work permit that will be a related resource? - // assume pump1 exists and will produce a timeseries RPM_pump1 with associated events - // like "Pump1 RPM below threshold" - // when the work permit is activated pump1 will be updated indicating it has an active work permit on it - // the produced event will have a related resource /source field indicating its source ie pump1 - // using the related resource filter we can filter on events that dont have an active work permit? - // can probalby do a lot more complex stuff aswell. - filter: Filter, - relation_types: Option>, -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -/// -/// Constructs an arbitrary boolean statement for advanced filtering. -/// -/// # Variants -/// -/// - And(Vec): Combines 2 or more filters on logical and. -/// -/// - Or(Vec): Combines 2 or more filters on logical or. -/// -/// - Not(Box): Negates the result of the child -/// -/// - Equals: Porperty value must match the given value exactly. -/// - property: The property name to be evaluated. -/// - value: The value to match against the property. -/// -/// - In: property matches any of a list of values. -/// - property: The property name to be evaluated. -/// - values: The list of values to check for inclusion. -/// -/// - Range: check if property is between min and max values -/// - property: The property name to be evaluated. -/// - min: The optional lower bound of the range (inclusive or exclusive support could be added in the future). -/// - max: The optional upper bound of the range (inclusive or exclusive support could be added in the future). -/// -/// - IsSet: check listed properties are not None -/// - property: A vector of property names to check. -/// -/// - ContainsAny: property (list-type) contains at least 1 element in any_of -/// - property: The property name to be evaluated. -/// - any_of: The list of values to check for presence. -/// -/// - ContainsAll: property (list-type) contains all elements in all_of -/// - property: The property name to be evaluated. -/// - all_of: The list of values that must all be present. -/// -/// # Usage -/// -/// This filter can be used to construct an arbitrary boolean statement: -/// -/// ```rust -/// use serde_json; -/// use intellistream_datahub_sdk::{filters::Filter}; -/// use serde_json::json; -/// let filter = Filter::And(vec![ -/// Filter::Equals { -/// property: "status".to_string(), -/// value: "active".to_string() -/// }, -/// Filter::Not(Box::new(Filter::In { -/// property: "category".to_string(), -/// values: vec!["restricted".to_string(), "archived".to_string()] -/// })), -/// ]); -/// let serialized = serde_json::to_string(&filter).unwrap(); -/// println!("{}", serialized); -/// let deserialized: Filter = serde_json::from_str(&serialized).unwrap(); -/// assert_eq!(filter, deserialized); -///``` -/// // -#[serde(rename_all = "camelCase")] -pub enum Filter { - // filters are constructed as a tree like structure using the standard boolean operators AND, OR, NOT, - And(Vec), - Or(Vec), - Not(Box), - // these filters assume the property is a single value - Equals { - // evaluate if the given property is equal to the given value - property: String, - value: String, - }, - In { - // evaluate if any of the given values are equal to the given property - property: String, - values: Vec, - }, - Range { - // evaluate if the given property is between the given min and max values - // in future we could add support for inclusive/exclusive bounds - max: Option, - min: Option, - property: String, - }, - IsSet { - // evaluate if the given property is set (not null) - property: Vec, - }, - // the remaining filters assume a property is a list - #[serde(rename_all = "camelCase")] - ContainsAny { - // check if the property contains any of the given values - any_of: Vec, - property: String, - }, - #[serde(rename_all = "camelCase")] - ContainsAll { - all_of: Vec, - property: String, - }, -} - -impl Filter { - pub fn and(filters: &Vec) -> Self { - Filter::And(filters.clone()) - } - pub fn or(filters: &Vec) -> Self { - Filter::Or(filters.clone()) - } - pub fn not(filter: &Filter) -> Self { - Filter::Not(Box::new(filter.clone())) - } - pub fn eq(property: &str, value: &str) -> Self { - Filter::Equals { - property: property.to_string(), - value: value.to_string(), - } - } - pub fn in_values(property: &str, values: &Vec) -> Self { - Filter::In { - property: property.to_string(), - values: values.clone(), - } - } - pub fn range(property: &str, min: Option, max: Option) -> Self { - Filter::Range { - property: property.to_string(), - min, - max, - } - } - pub fn is_set(property: &Vec) -> Self { - Filter::IsSet { - property: property.clone(), - } - } - pub fn contains_any(property: &str, any_of: &[String]) -> Self { - Filter::ContainsAny { - property: property.to_string(), - any_of: any_of.to_vec(), - } - } - pub fn contains_all(property: &str, all_of: &[String]) -> Self { - Filter::ContainsAll { - property: property.to_string(), - all_of: all_of.to_vec(), - } - } - - pub fn build(&self) -> Self { - self.clone() - } -} #[cfg(test)] mod tests { use super::*; use serde_json::json; - #[test] - fn test_advanced_filter() { - let mut filt = AdvancedEventFilter::new(); - assert_eq!(filt.filter, None); - let leaf_filter1 = Filter::Equals { - property: "test".to_string(), - value: "test".to_string(), - }; - let leaf_filter2 = Filter::contains_all( - "policies", - &vec!["policy1".to_string(), "policy2".to_string()], - ); - let leaf_filter3 = Filter::is_set(&vec!["metdata".to_string(), "type".to_string()]); - let leaf_filter4 = Filter::contains_any( - "policies", - &vec!["policy3".to_string(), "policy4".to_string()], - ); - - let bool_filter1 = Filter::and(&vec![leaf_filter1.clone(), leaf_filter2.clone()]); - let bool_filter2 = Filter::or(&vec![leaf_filter2, leaf_filter4]); - let bool_filter3 = Filter::not(&leaf_filter3); - let expected_json = json!({"filter": {"equals": {"property": "test", "value": "test"}}}); - filt.filter = Some(leaf_filter1.clone()); - assert_eq!( - serde_json::to_string(&filt).unwrap(), - expected_json.to_string() - ); - - let expected_json2 = json!( - {"filter": - {"and":[ - {"equals":{"property": "test", "value": "test"}}, - {"containsAll":{"property":"policies","allOf":["policy1", "policy2"]}} - ] - }}); - assert_eq!( - serde_json::to_string(&AdvancedEventFilter { - filter: Some(bool_filter1.clone()), - related_resource_filter: None - }) - .unwrap(), - expected_json2.to_string() - ); - let expected_json3 = json!({ - "filter": { - "and": [ - { - "or": [ - { - "and": [ - {"equals": {"property": "test", "value": "test"}}, - {"containsAll": {"property": "policies", "allOf": ["policy1", "policy2"]}} - ] - }, - { - "or": [ - {"containsAll": {"property": "policies", "allOf": ["policy1", "policy2"]}}, - {"containsAny": {"property": "policies", "anyOf": ["policy3", "policy4"]}} - ] - } - ] - }, - { - "not": { - "not": { - "isSet": {"property": ["metdata", "type"]} - } - } - } - ] - } - }); - - let filter1_or_2_and_not3 = Filter::and(&vec![ - Filter::or(&vec![bool_filter1.clone(), bool_filter2]), - Filter::not(&bool_filter3), - ]); - assert_eq!( - serde_json::to_string(&AdvancedEventFilter { - filter: Some(filter1_or_2_and_not3), - related_resource_filter: None - }) - .unwrap(), - expected_json3.to_string() - ) - } - - // The backend event filter reads a single `relatedResources: [{id}|{externalId}]` array - // (`Collection`), NOT flat `relatedResourceIds` / `relatedResourceExternalIds` - // arrays. Serializing the flat keys silently disabled related-resource filtering, so lock the - // wire shape here. #[test] fn basic_event_filter_serializes_related_resources_as_id_collection() { let mut filter = EventFilter::default(); @@ -941,16 +666,17 @@ mod tests { } /// `advanced_filter` used to serialize under its snake_case Rust name, which the api does not - /// read — the filter was silently ignored and the query came back unfiltered. + /// know. It is `advancedFilter` on the wire, and now carries an expression rather than a tree. #[test] fn advanced_filter_serializes_as_camel_case() { - let mut filter = EventFilterForm::default(); - filter.set_advanced_filter(AdvancedEventFilter::new()); + let mut form = EventFilterForm::default(); + form.set_advanced_filter("type = 'Alarm' AND subType IS NOT NULL"); + let value = serde_json::to_value(form.build()).unwrap(); - let value = serde_json::to_value(filter.build()).unwrap(); - assert!( - value.get("advancedFilter").is_some(), - "expected advancedFilter, got: {value}" + assert_eq!( + value.get("advancedFilter").and_then(|v| v.as_str()), + Some("type = 'Alarm' AND subType IS NOT NULL"), + "expected the expression under advancedFilter, got: {value}" ); assert!(value.get("advanced_filter").is_none()); }