docs: make the docs.rs reference usable

.github/workflows/ci.yml

@@ -36,6 +36,8 @@ jobs:
       # without failing a release. `blocking` did exactly that.
       - run: cargo build --all-features
       - run: cargo test --no-run --all-features
+      # Only --doc: every other test needs a live backend.
+      - run: cargo test --doc --all-features
       - run: cargo publish --dry-run
 
   wheels:

Cargo.toml

@@ -20,6 +20,11 @@ exclude = [
 # Synchronous client (intellistream_datahub_sdk::blocking), same split as reqwest::blocking.
 blocking = []
 
+# docs.rs builds with default features only, which would leave `blocking` undocumented even
+# though the README points callers at it.
+[package.metadata.docs.rs]
+all-features = true
+
 [dependencies]
 maplit = "1"
 tokio = { version = "1", features = ["full"] }

src/assets/mod.rs

@@ -1,3 +1,18 @@
+//! Assets — the typed `/assets` family, and the [`Asset`] shape its reads
+//! answer with.
+//!
+//! [`AssetsService`] is reached as `api.assets`. Every call is the generic `/resources` pipeline
+//! with the `ASSET` discriminator pinned server-side, so the two families cannot drift on ACLs or
+//! status codes; what differs is that reads answer [`Asset`] rather than the
+//! polymorphic [`Node`], putting `geolocation` and `is_root` in reach without a
+//! match. Filtering and searching reuse the resource bodies, and a `node_type` set on either is
+//! *replaced* with `["asset"]` rather than merged.
+//!
+//! Two things to expect: [`list`](AssetsService::list) does not page, so narrow with
+//! [`filter`](AssetsService::filter) instead of raising `limit`; and
+//! [`update`](AssetsService::update) echoes [`Node`]s rather than `Asset`s,
+//! because an update may touch relations whose other end is a different node type.
+
 #[cfg(test)]
 mod tests;
 

src/blocking.rs

@@ -1,7 +1,14 @@
 //! Blocking (synchronous) client, mirroring the async API — the same split as
 //! `reqwest` / `reqwest::blocking`.
 //!
-//! Enable it with the `blocking` cargo feature. Every wrapper delegates to the async
+//! **Requires the `blocking` cargo feature**, which is off by default:
+//!
+//! ```toml
+//! [dependencies]
+//! intellistream-datahub-sdk = { version = "0.4", features = ["blocking"] }
+//! ```
+//!
+//! Every wrapper delegates to the async
 //! implementation on a dedicated Tokio runtime owned by the client, so behavior
 //! (durable buffering, retries, OAuth token refresh) is identical to the async API —
 //! there is exactly one implementation of each call.

src/datahub.rs

@@ -1,3 +1,21 @@
+//! Configuration and authentication. [`DataHubConfig`] holds the API base URL and the credentials
+//! every call is made with, and mints, caches and refreshes the bearer token behind them.
+//!
+//! Build one from the process environment and a local `.env` with [`DataHubConfig::from_env`], or
+//! from a named file with [`from_envfile`](DataHubConfig::from_envfile), then pass it to
+//! [`ApiService::new`](crate::ApiService::new). `BASE_URL` is always required; for credentials,
+//! either set `TOKEN` to a token you manage yourself — that one is never treated as expired and
+//! never refreshed — or give `CLIENT_ID`, `CLIENT_SECRET` and `TOKEN_URI` for the
+//! client-credentials flow, or the `ASSERTION*` variables for the RFC 7523 `jwt-bearer` exchange.
+//! The setters adjust the same values in code, and also switch on the durable ingest buffer
+//! ([`enable_buffering`](DataHubConfig::enable_buffering)).
+//!
+//! `openid` is **always** requested and `SCOPE` *adds* to it rather than replacing it, because the
+//! API resolves dataset grants by calling the identity provider's UserInfo endpoint with your own
+//! token — on a Keycloak Organizations realm that means `SCOPE=organization:<alias>`. Everything
+//! here fails with [`DataHubError`], never with
+//! [`ResponseError`](crate::http::ResponseError).
+
 use crate::errors::DataHubError;
 use chrono::{DateTime, Utc};
 use dotenv::from_path;
@@ -24,14 +42,19 @@ pub const DEFAULT_BUFFER_DIR: &str = ".datahub-spool";
 /// Default number of datapoint insert requests in flight at once. Each carries up to 100 000 points,
 /// which the api holds in memory while it parses them, so this bounds what one insert costs its heap.
 pub const DEFAULT_DATAPOINT_INSERT_PARALLELISM: usize = 4;
-/// Scope sent with a token request when none is configured. See [`OAuthConfig::effective_scope`].
+/// Scope sent with a token request when none is configured. `SCOPE` adds to this rather than
+/// replacing it, so `SCOPE=organization:acme` asks for `openid organization:acme`.
 pub const DEFAULT_SCOPE: &str = "openid";
 /// RFC 7523 grant type: exchange an externally-issued JWT assertion for a token.
 const JWT_BEARER_GRANT: &str = "urn:ietf:params:oauth:grant-type:jwt-bearer";
 /// RFC 7523 client-authentication type: authenticate the client itself with a JWT assertion
 /// (Keycloak "Signed JWT - Federated") instead of a client secret.
 const JWT_BEARER_CLIENT_ASSERTION: &str = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer";
 
+/// The credential half of [`DataHubConfig`]: which OAuth2 flow to use and what to send with it.
+///
+/// Populated from the environment by [`DataHubConfig::from_env`]; the `ASSERTION*` fields select
+/// the RFC 7523 `jwt-bearer` exchange in place of plain client credentials.
 #[derive(Default, Deserialize, Debug, Clone)]
 pub struct OAuthConfig {
     #[serde(alias = "CLIENT_ID")]
@@ -134,6 +157,12 @@ pub(crate) struct AuthState {
     pub token: Option<oauth2::basic::BasicTokenResponse>,
     pub expire_time: Option<DateTime<Utc>>,
 }
+/// Base URL, credentials and the cached bearer token — everything an [`ApiService`](crate::ApiService)
+/// needs to talk to one backend as one tenant.
+///
+/// Build it with [`from_env`](Self::from_env) or [`from_envfile`](Self::from_envfile), adjust it
+/// with the setters, then hand it to [`ApiService::new`](crate::ApiService::new). Cloning is cheap
+/// and shares the token cache.
 #[derive(Debug, Clone)]
 pub struct DataHubConfig {
     pub(crate) config: Arc<OAuthConfig>,
@@ -525,6 +554,10 @@ impl DataHubConfig {
         auth_state.expire_time = None;
     }
 
+    /// The current bearer token, minting or refreshing it if the cached one is missing or expired.
+    ///
+    /// A `TOKEN` supplied through the environment is returned as-is and never refreshed — it is
+    /// assumed to be managed outside the SDK.
     pub async fn get_api_token(&self) -> Result<String, DataHubError> {
         {
             // lock scope. read and if expired refresh token

src/datasets/mod.rs

@@ -1,3 +1,20 @@
+//! Data sets — the container the rest of the platform hangs off, and the unit access is granted on.
+//!
+//! [`DatasetsService`] is reached as `api.datasets` and covers create, list, by-ids, filter,
+//! search, update and delete, all over [`Dataset`]. [`list`](DatasetsService::list) is a capped,
+//! unpaged sample of the tenant; criteria, ordering and cursor paging live on
+//! [`filter`](DatasetsService::filter).
+//!
+//! Because a data set is what read and write grants attach to, editing one is an operator action:
+//! [`update`](DatasetsService::update) requires the all-datasets write grant and answers **403**
+//! without it, even for a caller who may write the data set's *contents*. Two fields never come
+//! back populated — `connected_data_sets` is input-only, and `data_set_id` is always `None`, since
+//! a data set cannot belong to another one.
+//!
+//! [`policies`](DatasetsService::policies) is wired to the documented endpoint but has been
+//! observed answering with an empty body even where policies exist, so treat its result as
+//! unreliable rather than authoritative.
+
 #[cfg(test)]
 mod tests;
 
@@ -15,6 +32,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::Weak;
 
+/// Data set CRUD, filtering and search. Reached as `api.datasets`; see the [module docs](self).
 pub struct DatasetsService {
     pub(crate) api_service: Weak<ApiService>,
     base_url: String,
@@ -34,6 +52,9 @@ impl DatasetsService {
         }
     }
 
+    /// `POST /datasets/create` — create one or more data sets.
+    ///
+    /// Accepts a [`Dataset`], a `Vec` of them, or a reference to either.
     pub async fn create<I>(&self, data: &I) -> Result<DataWrapper<Dataset>, ResponseError>
     where
         for<'a> &'a I: Into<DataWrapper<Dataset>>,
@@ -44,6 +65,10 @@ impl DatasetsService {
             .await
     }
 
+    /// `POST /datasets/delete` — delete data sets by id or external id.
+    ///
+    /// A data set stands above everything that belongs to it, so this is refused while it still has
+    /// members.
     pub async fn delete<I>(&self, json: &I) -> Result<DataWrapper<Dataset>, ResponseError>
     where
         for<'a> &'a I: Into<DataWrapper<IdAndExtId>>,
@@ -84,6 +109,9 @@ impl DatasetsService {
         self.execute_post_request(path, filter).await
     }
 
+    /// `POST /datasets/byids` — fetch data sets by id or external id.
+    ///
+    /// Like every batch lookup, this answers 200 with the subset it found and silently omits the rest.
     pub async fn by_ids<I>(&self, id_collection: &I) -> Result<DataWrapper<Dataset>, ResponseError>
     where
         for<'a> &'a I: Into<DataWrapper<IdAndExtId>>,
@@ -166,6 +194,10 @@ impl DatasetsService {
     }
 }
 
+/// A data set: the container entities belong to, and the unit read and write grants attach to.
+///
+/// `connected_data_sets` is input-only and `data_set_id` is always `None` on a read — a data set
+/// cannot belong to another one.
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct Dataset {

src/errors.rs

@@ -1,3 +1,11 @@
+//! [`DataHubError`], the error type for configuration and authentication — what can go wrong
+//! *before* a request is made.
+//!
+//! [`DataHubConfig::from_env`](crate::datahub::DataHubConfig::from_env) and its siblings return it
+//! for a missing or unparseable setting, and the token exchange returns it for an OAuth2 or URL
+//! failure. Service methods never surface it: the SDK maps a failed token acquisition to a 401
+//! [`ResponseError`](crate::http::ResponseError) so every API call has one error type.
+
 use thiserror::Error;
 
 #[derive(Error, Debug)]

src/events/mod.rs

@@ -1,3 +1,32 @@
+//! Events — time-stamped records of something that happened, plus the vocabulary endpoints behind
+//! their filter dropdowns.
+//!
+//! [`EventsService`] is reached as `api.events`. An [`Event`] is identified by a UUID rather than a
+//! numeric id, and [`create`](EventsService::create) stamps a UUID v7 client-side on any event that
+//! arrives without one — so a retry carries the same id and the server collapses the duplicate
+//! instead of storing it twice.
+//!
+//! - **CRUD and queries** — [`create`](EventsService::create), [`get`](EventsService::get),
+//!   [`by_ids`](EventsService::by_ids), [`update`](EventsService::update),
+//!   [`delete`](EventsService::delete), plus [`filter`](EventsService::filter),
+//!   [`search`](EventsService::search) and [`count`](EventsService::count).
+//! - **Vocabulary** — [`list_types`](EventsService::list_types) /
+//!   [`search_types`](EventsService::search_types) and the same pair for sub-types, statuses and
+//!   sources (see [`EventDimension`]). They answer "what values does this tenant actually use" for
+//!   the four categorical fields, reading small server-side dimension tables instead of scanning
+//!   events — cheap, but only *eventually consistent* with the events themselves.
+//!
+//! Two surprises worth knowing. [`list`](EventsService::list) returns the **oldest** events, not
+//! the newest: it is `filter` with an empty body, whose default sort is `eventTime` ascending, so
+//! ask "what just happened" through `filter` with a descending sort. And [`EventUpdate`] cannot
+//! change `event_time` or `external_id` — both identify an event rather than describe it, and
+//! sending either is a 400; record a correction by creating a new event and deleting the old one.
+//!
+//! With durable buffering enabled on the client, [`create`](EventsService::create) can answer
+//! **202 with no items** because the batch was spooled to disk rather than sent. Check
+//! [`get_http_status_code`](crate::generic::DataWrapper::get_http_status_code) or
+//! [`buffered_count`](EventsService::buffered_count) when you need to tell the two apart.
+
 #[cfg(test)]
 mod tests;
 
@@ -17,6 +46,8 @@ use std::collections::HashMap;
 use std::sync::{Mutex, Weak};
 use uuid::Uuid;
 
+/// Event CRUD, filtering, search and the vocabulary endpoints. Reached as `api.events`; see the
+/// [module docs](self).
 pub struct EventsService {
     pub(crate) api_service: Weak<ApiService>,
     base_url: String,
@@ -34,6 +65,13 @@ impl EventsService {
         }
     }
 
+    /// `POST /events/create` — create one or more events.
+    ///
+    /// Any event without an `id` is given a time-ordered UUID v7 before sending, so a retry carries
+    /// the same id and the server collapses the duplicate rather than storing it twice.
+    ///
+    /// With durable buffering enabled, a batch that cannot reach the server is spooled to disk and
+    /// this answers **202 with no items** instead of failing.
     pub async fn create<I>(&self, data: &I) -> Result<DataWrapper<Event>, ResponseError>
     where
         for<'a> &'a I: Into<DataWrapper<Event>>,
@@ -160,6 +198,7 @@ impl EventsService {
         }
     }
 
+    /// `POST /events/delete` — delete events by UUID.
     pub async fn delete<I>(&self, json: &I) -> Result<DataWrapper<Event>, ResponseError>
     where
         for<'a> &'a I: Into<DataWrapper<EventIdCollection>>,
@@ -191,11 +230,15 @@ impl EventsService {
             .await
     }
 
+    /// `POST /events/filter` — events matching [`EventFilterForm`], with sorting and keyset paging.
+    ///
+    /// Defaults to `eventTime` ascending. Echo the response's `next_cursor` back to walk the rest.
     pub async fn filter(&self, filter: &EventFilterForm) -> Result<DataWrapper<Event>, ResponseError> {
         let path = &format!("{}/filter", self.base_url);
         self.execute_post_request(path, &filter).await
     }
 
+    /// `POST /events/byids` — fetch events by UUID, answering the subset it found.
     pub async fn by_ids<I>(&self, id_collection: &I) -> Result<DataWrapper<Event>, ResponseError>
     where
         for<'a> &'a I: Into<DataWrapper<EventIdCollection>>,
@@ -435,6 +478,10 @@ fn buffered_wrapper() -> DataWrapper<Event> {
 // Not PartialEq: it carries `Vec<IdAndExtId>`, which is intentionally non-comparable (see
 // `IdAndExtId`) — the same resource can be named as {id}, {externalId} or both. Same reasoning as
 // `EventFilter`.
+/// Something that happened, at a time.
+///
+/// Identified by a UUID rather than a numeric id. `external_id` is deliberately *not* unique: the
+/// events sharing one are the lifecycle of a single logical event.
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct Event {

src/fields.rs

@@ -1,6 +1,22 @@
+//! The update primitives the `POST /<collection>/update` request bodies are built from:
+//! [`Field`] for a scalar, [`ListField`] for a list, [`MapField`] for `metadata`.
+//!
+//! An update names its target and then describes only the fields it changes, so each of these has
+//! to distinguish "leave alone" (absent) from "set to this" and from "clear". Set or clear a
+//! scalar with [`Field::value`] and [`Field::null`]. Lists and maps go further: [`ListField::set`]
+//! and [`MapField::set`] replace the whole collection, while `add` / `remove` apply a delta and
+//! leave the rest in place — the two are mutually exclusive by construction, so a body carrying
+//! both cannot be built.
+//!
+//! You meet these as the fields of each entity's update form, e.g.
+//! [`ResourceUpdateFields`](crate::resources::ResourceUpdateFields).
+
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
+/// One scalar field of an update: absent (leave alone), set to a value, or explicitly cleared.
+///
+/// Build with [`Field::value`] or [`Field::null`]; the default is absent.
 #[derive(Debug, Deserialize, Serialize, Clone, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct Field<T> {

src/files/mod.rs

@@ -1,3 +1,20 @@
+//! Files — upload, download, and the directory tree they live in.
+//!
+//! [`FileService`] is reached as `api.files`. Files and folders are both
+//! [`INode`]s: walk the tree with
+//! [`list_root_directory`](FileService::list_root_directory) and
+//! [`list_directory_by_path`](FileService::list_directory_by_path), look one up with
+//! [`get_by_id`](FileService::get_by_id), [`get_by_external_id`](FileService::get_by_external_id)
+//! or [`search`](FileService::search), rename, move or re-assign a data set with
+//! [`update`](FileService::update), and soft-delete, [`list_trash`](FileService::list_trash) and
+//! [`restore`](FileService::restore).
+//!
+//! Upload is a raw `PUT`: the file content is the request body and all metadata rides in
+//! `X-Datahub-*` headers, which [`FileUpload`] builds for you — but note that [`FileUpload::new`]
+//! inspects the path eagerly and **panics** if it is not a readable regular file, so validate it
+//! first. Download comes in two shapes: [`download`](FileService::download) buffers the whole file
+//! into memory, while [`download_to_path`](FileService::download_to_path) streams it to disk.
+
 mod test;
 
 use crate::datahub::to_snake_lower_cased_allow_start_with_digits;
@@ -15,6 +32,7 @@ use tokio::fs::File;
 use tokio::io::AsyncWriteExt;
 use tokio_util::codec::{BytesCodec, FramedRead};
 
+/// File upload, download and the directory tree. Reached as `api.files`; see the [module docs](self).
 pub struct FileService {
     pub(crate) api_service: Weak<ApiService>,
     base_url: String,
@@ -29,6 +47,10 @@ impl FileService {
         }
     }
 
+    /// `PUT /files` — upload a file's contents, with its metadata in `X-Datahub-*` headers.
+    ///
+    /// The body is the raw bytes rather than a multipart form. Build the [`FileUpload`] first; it
+    /// reads the path eagerly and panics if it is not a readable regular file.
     pub async fn upload_file(
         &self,
         file_upload: FileUpload,
@@ -41,13 +63,15 @@ impl FileService {
             .await
     }
 
+    /// The top level of the file tree, as [`INode`]s — files and folders alike.
     pub async fn list_root_directory(&self) -> Result<DataWrapper<INode>, ResponseError> {
         // Create and send an HTTP GET request
         let full_path = format!("{}/list", self.base_url.as_str());
         self.execute_get_request(full_path.as_str(), None::<&str>)
             .await
     }
 
+    /// The contents of one folder, named by its path.
     pub async fn list_directory_by_path(
         &self,
         path: &str,
@@ -57,6 +81,10 @@ impl FileService {
             .await
     }
 
+    /// `POST /files/delete` — move files to the trash by id or external id.
+    ///
+    /// Soft: see [`list_trash`](Self::list_trash) for what is in there and
+    /// [`restore`](Self::restore) to bring one back.
     pub async fn delete(
         &self,
         id_collection: &DataWrapper<IdAndExtId>,
@@ -341,6 +369,10 @@ impl FileUpdate {
     }
 }
 
+/// A file staged for upload: the local path plus the metadata that rides in `X-Datahub-*` headers.
+///
+/// [`new`](Self::new) inspects the path immediately and **panics** if it is not a readable regular
+/// file, so check it first if the path came from outside your program.
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct FileUpload {
     #[serde(rename = "externalId")]