diff --git a/.github/workflows/apply-httproute.yaml b/.github/workflows/apply-httproute.yaml new file mode 100644 index 0000000..6a2de75 --- /dev/null +++ b/.github/workflows/apply-httproute.yaml @@ -0,0 +1,38 @@ +name: apply-httproute + +on: + workflow_dispatch: + +jobs: + httproute: + name: apply httproute + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Configure kubeconfig + env: + KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} + run: | + mkdir -p ~/.kube + echo "${KUBECONFIG_B64}" | base64 -d > ~/.kube/config + + - name: Apply HTTPRoute + run: | + set -euo pipefail + kubectl apply --dry-run=server -f deployment/httproute.yaml -n website + kubectl apply -f deployment/httproute.yaml -n website + + - name: Wait for route to be accepted + run: | + set -euo pipefail + for route in docs docs-redirect; do + for cond in Accepted ResolvedRefs; do + kubectl wait --timeout=60s -n website "httproute/$route" \ + --for=jsonpath="{.status.parents[0].conditions[?(@.type==\"$cond\")].status}"=True + done + done diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b06d5b5..523bd3a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -130,38 +130,3 @@ jobs: --default-repo=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} \ --build-artifacts build.out.json \ -m ${{ env.MODULE }} - - httproute: - name: apply httproute - needs: deploy - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Configure kubeconfig - env: - KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }} - run: | - mkdir -p ~/.kube - echo "${KUBECONFIG_B64}" | base64 -d > ~/.kube/config - - - name: Apply HTTPRoute - run: | - set -euo pipefail - kubectl apply --dry-run=server -f deployment/httproute.yaml -n website - kubectl apply -f deployment/httproute.yaml -n website - - - name: Wait for route to be accepted - run: | - set -euo pipefail - for route in docs docs-redirect; do - for cond in Accepted ResolvedRefs; do - kubectl wait --timeout=60s -n website "httproute/$route" \ - --for=jsonpath="{.status.parents[0].conditions[?(@.type==\"$cond\")].status}"=True - done - done diff --git a/docs/UML/drawio/hexagonal-architecture.drawio b/docs/UML/drawio/hexagonal-architecture.drawio index db50ce1..4062dfb 100644 --- a/docs/UML/drawio/hexagonal-architecture.drawio +++ b/docs/UML/drawio/hexagonal-architecture.drawio @@ -1,10 +1,10 @@ - + - + @@ -118,6 +118,24 @@ + + + + + + + + + + + + + + + + + + diff --git a/docs/api-reference/injection/index.md b/docs/api-reference/injection/index.md index 5eaaa51..7be933d 100644 --- a/docs/api-reference/injection/index.md +++ b/docs/api-reference/injection/index.md @@ -4,6 +4,7 @@ options: members: [] show_root_heading: false + show_root_toc_entry: false Each registry below resolves an implementation **by name, on first use**. diff --git a/docs/api-reference/namespaces.md b/docs/api-reference/namespaces.md index d3fb2bb..18d8188 100644 --- a/docs/api-reference/namespaces.md +++ b/docs/api-reference/namespaces.md @@ -1,13 +1,19 @@ # Import Namespaces +These are the important namespaces in the repository. + | Namespace | Description | | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `hololinked.core` | Definition of [`Thing`](thing/index.md) class, [`Property`](property/index.md), [`Action`](action/index.md) and [`Event`](events/index.md) classes, [`StateMachine`](state-machine/state-machine.md), [`Logger`](thing/logger.md) & the [`EventLoop`](eventloop/eventloop.md) that executes operations. | -| `hololinked.serializers` | Serializers for different data formats (JSON, MessagePack, etc.) and [`Serializers`](serializers/index.md) singleton. | +| `hololinked.serializers` | Serializers for different data formats (JSON, MessagePack, etc.) | | `hololinked.server` | Protocol specific server implementations | -| `hololinked.client` | [`ObjectProxy`](clients/object-proxy.md) and protocol specific client implementations, client abstractions and [`ClientFactory`](clients/base.md). | -| `hololinked.param` | Copy of `param` library with implementation specific modifications. | +| `hololinked.client` | [`ObjectProxy`](clients/object-proxy.md), protocol specific client implementations, client abstractions and [`ClientFactory`](clients/base.md). | +| `hololinked.param` | Copy of [`holoviz param`](https://param.holoviz.org/en/docs/latest/index.html) library with implementation specific modifications. | | `hololinked.storage` | Database clients for data persistence. | | `hololinked.metadata` | Device description languages a `Thing` can be described in. `hololinked.metadata.td` holds the W3C Web of Things Thing Description/[ThingModel](td/tm.md) parsing and generation. | -| `hololinked.injection` | Dependency injection layer. Holds the lazy adapter registries — `Serializers`, `SchemaValidators`, `StorageBackends` and `MetadataFormats` — that resolve an implementation by name on first use. | +| `hololinked.injection` | Dependency injection layer, holds the lazy adapter registries like `Serializers`, `SchemaValidators`, `StorageBackends` and `MetadataFormats` that resolve an implementation by name on first use. | | `hololinked.schema_validators` | Validators that check property, action and event payloads against a schema (JSON Schema, fastjsonschema, pydantic). | + +A [hexagonal architecture](../design/hexagonal-architecture.md) is followed with `hololinked.core` being the important part of the repository. + +Hexagonal architecture diagram \ No newline at end of file diff --git a/docs/api-reference/protocols/http/config.md b/docs/api-reference/protocols/http/config.md index 89e64ef..1cfb90c 100644 --- a/docs/api-reference/protocols/http/config.md +++ b/docs/api-reference/protocols/http/config.md @@ -20,7 +20,7 @@ - thing_description_service Pass these as a plain dictionary to the `config` argument of -[`HTTPServer`](index.md) — not as a `RuntimeConfig` instance: +[`HTTPServer`](index.md), not as a `RuntimeConfig` instance: ```python HTTPServer(config=dict(cors=True, property_handler=MyPropertyHandler)) diff --git a/docs/api-reference/protocols/zmq/messaging-contract/headers.md b/docs/api-reference/protocols/zmq/messaging-contract/headers.md index a4ff014..e199c5e 100644 --- a/docs/api-reference/protocols/zmq/messaging-contract/headers.md +++ b/docs/api-reference/protocols/zmq/messaging-contract/headers.md @@ -1,8 +1,13 @@ # Headers ::: hololinked.server.zmq.message.RequestHeader - + options: + heading_level: 2 + ::: hololinked.server.zmq.message.ResponseHeader + options: + heading_level: 2 ::: hololinked.server.zmq.message.EventHeader - + options: + heading_level: 2 diff --git a/docs/api-reference/server/index.md b/docs/api-reference/server/index.md index 2e1a301..34a54a3 100644 --- a/docs/api-reference/server/index.md +++ b/docs/api-reference/server/index.md @@ -3,14 +3,11 @@ ::: hololinked.server options: show_root_heading: false + show_root_toc_entry: false members: false ## Entry points -`run()` and `stop()` knows no protocol in particular — they take already-constructed servers and -drive their lifecycle. Use them when you build servers yourself; `Thing.run(access_points=...)` is -the shorthand that calls `run()`. - ::: hololinked.server.run options: show_root_heading: true diff --git a/docs/assets/hexagonal-architecture.drawio.svg b/docs/assets/hexagonal-architecture.drawio.svg index 0bc229e..9f1d3ed 100644 --- a/docs/assets/hexagonal-architecture.drawio.svg +++ b/docs/assets/hexagonal-architecture.drawio.svg @@ -1,4 +1,4 @@ -
hololinked.core

Thing
Property, Action, Event
RPCServer, EventPublisher
Serializers
Metadata Generator
(ThingModel generator)
storage & database
(JSON, sqlalchemy, mongo)
Protocol Servers
(HTTP, MQTT etc.)
Driven Dependencies
Driving
Dependencies
Payload Validators
(Schema based)
 \ No newline at end of file +
hololinked.core

Thing
Property, Action, Event
EventLoop, EventPublisher
Serializers
Metadata Generator
(ThingModel generator)
storage & database
(JSON, sqlalchemy, mongo)
Protocol Servers
(HTTP, MQTT etc.)
Driven Dependencies
Driving
Dependencies
Payload Validators
(Schema based)
BaseSchemaValidator
interface
BaseConfigurationRepository
interface
Metadata
interface
BaseSerializer
interface
Request-Reply (EventLoop)
PubSub
(EventPublisher)
 \ No newline at end of file diff --git a/docs/design/eventloop.md b/docs/design/eventloop.md new file mode 100644 index 0000000..16f9456 --- /dev/null +++ b/docs/design/eventloop.md @@ -0,0 +1,32 @@ +# Event Loop + +The main purpose of the [`EventLoop`](../api-reference/eventloop/) is to: + +- add scheduling control for the interaction affordances +- decouple protocol handlers from the actual execution of the interaction affordances +- unify execution across multiple protocols +- maintain standardized behaviour irrespective of the protocol, for example, pushing an event must cause both a HTTP and a MQTT client to receive the same event + +Three types of scheduling are supported: + +- **Synchronous** – the default, blocking until the interaction affordance is completed. +- **Async** – non-blocking and can be scheduled on the current async loop. +- **Threaded** – non-blocking and can be scheduled on a separate thread. + +All supported protocols retrieve information from their request object, which is used to create an [`Operation`](../api-reference/eventloop/operation.md). The `EventLoop` is invoked with the `Operation`, which handles the scheduling and execution of the interaction affordance. The response is sent back to the protocol handler, which returns the response to the client: + +```mermaid +sequenceDiagram + participant Client + participant ProtocolHandler + participant EventLoop + + Client->>ProtocolHandler: Send Request + ProtocolHandler->>EventLoop: Schedule an Operation + EventLoop->>EventLoop: Schedule & Execute Affordance (in its own thread) + EventLoop->>ProtocolHandler: Write Response + ProtocolHandler->>Client: Return Response + Note over EventLoop: --- Parallel Event Flow (Pub/Sub) --- + EventLoop-->>ProtocolHandler: Publish Event (Pub) + ProtocolHandler-->>Client: Forward Event (Pub, protocol specific) +``` \ No newline at end of file diff --git a/docs/design/zmq.md b/docs/design/zmq.md deleted file mode 100644 index 65a41c5..0000000 --- a/docs/design/zmq.md +++ /dev/null @@ -1,160 +0,0 @@ -# ZMQ RPC Broker - -The main purpose of the RPC broker is to: - -- add scheduling control for the interaction affordances -- decouple protocol handlers (a.k.a controllers) from the actual execution of the interaction affordances -- unify execution across multiple protocols -- maintain standardized behaviour irrespective of the protocol, for example, pushing an event must cause both a HTTP and a MQTT client to receive the same event - -Three types of scheduling are supported: - -- **Synchronous** – the default, blocking until the interaction affordance is completed. -- **Async** – non-blocking and can be scheduled on the current async loop. -- **Threaded** – non-blocking and can be scheduled on a separate thread. - -All supported protocols retrieve information from their request object, which is used to create a scheduling message. The message is then sent to the ZMQ RPC Broker, which handles the scheduling and execution of the interaction affordance. The ZMQ server then sends the response back to the protocol handler, which returns the response to the client: - -```mermaid -sequenceDiagram - participant Client - participant ProtocolHandler - participant ZMQServer - participant SharedMemory as SharedMemory(INPROC) - - Client->>ProtocolHandler: Send Request - ProtocolHandler->>SharedMemory: Write Scheduling Message - SharedMemory->>ZMQServer: Receive New Message
(in its own thread,
always non blocking to
receive all messages) - ZMQServer->>ZMQServer: Schedule & Execute Affordance - ZMQServer->>SharedMemory: Write Response - SharedMemory->>ProtocolHandler: Forward Response - ProtocolHandler->>Client: Return Response - Note over ZMQServer,SharedMemory: --- Parallel Event Flow (Pub/Sub) --- - ZMQServer-->>SharedMemory: Publish Event (Pub) - SharedMemory-->>ProtocolHandler: Forward Event (Sub) - ProtocolHandler-->>Client: Forward Event (Pub, protocol specific) -``` - -In a certain way, this is definitely a protocol-to-protocol transfer of request and response, with a minor difference that ZMQ handles this over its `INPROC` transport over threads & shared memory, which is significantly more efficient than TCP-to-TCP communication & adds very little overhead. - -The scheduling message is a simple JSON object, tailored to arbitrary serialization and ZMQ multipart messaging scheme: - -=== "Request Message" - - ```json - { - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "description": "header of the message", - "properties": { - "senderID" : { - "type": "string", - "description": "id of the sender of the message" - }, - "receiverID" : { - "type": "string", - "description": "id of the receiver of the message" - }, - "messageType" : { - "type": "string", - "description": "type of the message being sent", - "enum": [ - "HANDSHAKE", "OPERATION", "EXIT", "INTERRUPT" - ] - }, - "messageID" : { - "type": "string", - "format": "uuid", - "description": "unique id of the message to correlate with the response" - }, - "serverExecutionContext" : { - "type": "object", - "description": "activities to be taken care by the server while performing the operation", - "properties": { - "oneway": { - "type": "boolean" - }, - "invokationTimeout": { - "type": "number", - "minimum": 0 - }, - "executionTimeout": { - "type": "number", - "minimum": 0 - } - } - }, - "thingID" : { - "type": "string", - "description": "id of the thing on which the operation is to be performed" - }, - "object" : { - "type": "string", - "description" : "a property, action or event - object of the thing on which the operation is to be performed" - }, - "operation" : { - "type": "string", - "description" : "name of the peration to be performed", - "enum" : [ - "readProperty", "writeProperty", "invokeAction", "subscribeEvent", - "unsubscribeEvent", "readMultipleProperties", "writeMultipleProperties" - ] - }, - "payloadContentType" : { - "type": "string", - "description" : "content type of the payload", - "enum" : [ - "application/json", "pickle", "x-msgpack", "text", "text/plain" - ] - }, - "preencodedPayloadContentType" : { - "type": "string", - "description" : "content type of the pre-encoded payload" - }, - "thingExecutionContext" : { - "type": "object", - "description" : "thing execution context while performing the operation", - "properties" : { - "fetchExecutionLogs" : { - "type" : "boolean" - } - } - } - }, - "required": ["messageType", "messageID", "serverExecutionContext"] - } - ``` - -=== "Response Message" - - ```json - { - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties" : { - "messageID" : { - "type": "string", - "format": "uuid", - "description": "unique id of the message to correlate with the request" - }, - "messageType" : { - "type": "string", - "description": "type of the message", - "enum": ["REPLY", "ERROR", "TIMEOUT", - "INVALID_MESSAGE", "EVENT_DISCONNECTED"] - }, - "payloadContentType" : { - "type": "string", - "description" : "content type of the payload", - "enum" : [ - "application/json", "pickle", "x-msgpack", "text", "text/plain" - ] - }, - "preencodedPayloadContentType" : { - "type": "string", - "description" : "content type of the pre-encoded payload" - } - }, - "required": ["messageType", "messageID"] - } - ``` diff --git a/docs/index.md b/docs/index.md index 30494b3..348c860 100644 --- a/docs/index.md +++ b/docs/index.md @@ -90,7 +90,7 @@ Additionally, **state machines** can constrain property and action execution: - _Oscilloscope_ - Cannot start a new measurement while one is ongoing - _Camera_ - Cannot change exposure time while capturing video -#### Benefits you get when you use hololinked: +## Benefits you get when you use hololinked - technology independent and technology extensible architecture -> handles technology changes. - its not a solution that uses "different set of technologies to solve the same problem". It solves a problem irrespective of the underlying technologies. AI included. You chose the tech stack or extend it with your stack (PRs welcome for new tech adapters and new features). diff --git a/docs/introduction/security-scanning.md b/docs/introduction/security-scanning.md index b1c1891..78709f3 100644 --- a/docs/introduction/security-scanning.md +++ b/docs/introduction/security-scanning.md @@ -1,6 +1,6 @@ # Scanning -## Bandit +## bandit You can always access the latest security scan results from the Bandit SAST tool in the CI pipeline job of the [main branch](https://github.com/hololinked-dev/hololinked/actions/workflows/ci-pipeline.yml?query=branch%3Amain). diff --git a/mkdocs.yml b/mkdocs.yml index ee1cf57..7800324 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -70,20 +70,15 @@ nav: - QueuedScheduler: api-reference/eventloop/schedulers/queued-scheduler.md - ThreadedScheduler: api-reference/eventloop/schedulers/threaded-scheduler.md - AsyncScheduler: api-reference/eventloop/schedulers/async-scheduler.md - - Clients: - - ClientFactory: api-reference/clients/base.md - - ObjectProxy: api-reference/clients/object-proxy.md - - Abstractions: - - ConsumedThingAction: api-reference/clients/bases/cta.md - - ConsumedThingProperty: api-reference/clients/bases/ctp.md - - ConsumedThingEvent: api-reference/clients/bases/cte.md - - SSE: api-reference/clients/bases/sse.md - - Utils: api-reference/clients/bases/utils.md - - Security: - - Basic: api-reference/clients/security/basic.md - - API Key: api-reference/clients/security/apikey.md - - OIDC Direct Access Grant: api-reference/clients/security/oidc.md + - Interfaces: + - Ports: api-reference/interfaces/index.md + - BaseSerializer: api-reference/interfaces/serializer.md + - BaseSchemaValidator: api-reference/interfaces/schema-validator.md + - BaseConfigurationRepository: api-reference/interfaces/configuration-repository.md + - Metadata: api-reference/interfaces/metadata.md + - BaseProtocolServer: api-reference/interfaces/protocol-server.md - Servers: + - run: api-reference/server/index.md - HTTP: - HTTPServer: api-reference/protocols/http/index.md - Runtime Configuration: api-reference/protocols/http/config.md @@ -120,21 +115,19 @@ nav: - Security: - Bcrypt: api-reference/protocols/security/bcrypt.md - Argon2: api-reference/protocols/security/argon2.md - - Server: api-reference/server/index.md - - Configuration: api-reference/global-config.md - - Interfaces: - - Ports: api-reference/interfaces/index.md - - BaseSerializer: api-reference/interfaces/serializer.md - - BaseSchemaValidator: api-reference/interfaces/schema-validator.md - - BaseConfigurationRepository: api-reference/interfaces/configuration-repository.md - - Metadata: api-reference/interfaces/metadata.md - - BaseProtocolServer: api-reference/interfaces/protocol-server.md - - Dependency Injection: - - Adapter Registries: api-reference/injection/index.md - - SchemaValidators: api-reference/injection/schema-validators.md - - StorageBackends: api-reference/injection/storage-backends.md - - MetadataFormats: api-reference/injection/metadata-formats.md - - ProtocolServers: api-reference/injection/protocol-servers.md + - Clients: + - ClientFactory: api-reference/clients/base.md + - ObjectProxy: api-reference/clients/object-proxy.md + - Abstractions: + - ConsumedThingAction: api-reference/clients/bases/cta.md + - ConsumedThingProperty: api-reference/clients/bases/ctp.md + - ConsumedThingEvent: api-reference/clients/bases/cte.md + - SSE: api-reference/clients/bases/sse.md + - Utils: api-reference/clients/bases/utils.md + - Security: + - Basic: api-reference/clients/security/basic.md + - API Key: api-reference/clients/security/apikey.md + - OIDC Direct Access Grant: api-reference/clients/security/oidc.md - Schema Validators: - JSONSchemaValidator: api-reference/schema-validators/json-schema.md - FastJSONSchemaValidator: api-reference/schema-validators/fast-json-schema.md @@ -154,19 +147,27 @@ nav: - BaseDB: api-reference/storage/bases.md - Configuration: api-reference/storage/config.md - Models: api-reference/storage/models.md - # - Thing Descriptions: - # - Interaction Affordances: - # - InteractionAffordance: api-reference/td/interaction_affordance/interaction_affordance.md - # - PropertyAffordance: api-reference/td/interaction_affordance/property_affordance.md - # - ActionAffordance: api-reference/td/interaction_affordance/action_affordance.md - # - EventAffordance: api-reference/td/interaction_affordance/event_affordance.md - # - DataSchema: api-reference/td/data_schema.md - # - Thing Model: api-reference/td/tm.md - # - Form: api-reference/td/form.md - # - Metadata: api-reference/td/metadata.md - # - Security Definitions: api-reference/td/security.md - # - Bases: - # - Schema: api-reference/td/bases/schema.md + - Metadata/API Formats: + - Thing Descriptions: + - Interaction Affordances: + - InteractionAffordance: api-reference/td/interaction_affordance/interaction_affordance.md + - PropertyAffordance: api-reference/td/interaction_affordance/property_affordance.md + - ActionAffordance: api-reference/td/interaction_affordance/action_affordance.md + - EventAffordance: api-reference/td/interaction_affordance/event_affordance.md + - DataSchema: api-reference/td/data_schema.md + - Thing Model: api-reference/td/tm.md + - Form: api-reference/td/form.md + - Metadata: api-reference/td/metadata.md + - Security Definitions: api-reference/td/security.md + - Bases: + - Schema: api-reference/td/bases/schema.md + - Configuration: api-reference/global-config.md + - Dependency Injection: + - Adapter Registries: api-reference/injection/index.md + - SchemaValidators: api-reference/injection/schema-validators.md + - StorageBackends: api-reference/injection/storage-backends.md + - MetadataFormats: api-reference/injection/metadata-formats.md + - ProtocolServers: api-reference/injection/protocol-servers.md # Examples # - Examples: # - Energy Meter: examples/energy-meter.md @@ -185,7 +186,7 @@ nav: - Metaclasses: design/metaclasses.md - State Machine: design/state-machine.md - Serialization: design/serialization.md - - ZMQ RPC Layer: design/zmq.md + - Event Loop: design/eventloop.md - Hexagonal Architecture: design/hexagonal-architecture.md theme: