diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ebefa081..a3b0312b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,40 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [v2.0.2](https://github.com/oceanprotocol/ocean-node/compare/v2.0.1...v2.0.2) + +- document publish flow & isolated markets [`#1273`](https://github.com/oceanprotocol/ocean-node/pull/1273) +- make sure we bill all cases [`#1277`](https://github.com/oceanprotocol/ocean-node/pull/1277) + +#### [v2.0.1](https://github.com/oceanprotocol/ocean-node/compare/v2.0.0...v2.0.1) + +> 23 March 2026 + +- fix: payments claim issue [`#1275`](https://github.com/oceanprotocol/ocean-node/pull/1275) +- fix: remove some fields from getJobs handler response [`#1271`](https://github.com/oceanprotocol/ocean-node/pull/1271) +- getFile: return proper type [`#1267`](https://github.com/oceanprotocol/ocean-node/pull/1267) +- refactor fileinfo route & handler [`#1265`](https://github.com/oceanprotocol/ocean-node/pull/1265) +- Release 2.0.1 [`93740ac`](https://github.com/oceanprotocol/ocean-node/commit/93740acd3f5648484661b0fb65a92b236da6cb7e) + +### [v2.0.0](https://github.com/oceanprotocol/ocean-node/compare/v1.0.7...v2.0.0) + +> 16 March 2026 + +- Length prefixed streams [`#1232`](https://github.com/oceanprotocol/ocean-node/pull/1232) +- feat: update quickstart script & gpu docs [`#1261`](https://github.com/oceanprotocol/ocean-node/pull/1261) +- add ftp support & upload for url/ftp/s3. Reorg tests [`#1260`](https://github.com/oceanprotocol/ocean-node/pull/1260) +- Bump tar from 7.5.10 to 7.5.11 [`#1262`](https://github.com/oceanprotocol/ocean-node/pull/1262) +- Bump tar from 7.5.8 to 7.5.10 [`#1254`](https://github.com/oceanprotocol/ocean-node/pull/1254) +- storage cleanup & add S3 Storage for datasets (so far) [`#1256`](https://github.com/oceanprotocol/ocean-node/pull/1256) +- fix: integration c2d test [`#1212`](https://github.com/oceanprotocol/ocean-node/pull/1212) +- Release 2.0.0 [`0b852a1`](https://github.com/oceanprotocol/ocean-node/commit/0b852a1241d7dfdbf6db2540a6f3a513e2387111) + #### [v1.0.7](https://github.com/oceanprotocol/ocean-node/compare/v1.0.6...v1.0.7) +> 4 March 2026 + - fix query [`#1252`](https://github.com/oceanprotocol/ocean-node/pull/1252) +- Release 1.0.7 [`a3706e6`](https://github.com/oceanprotocol/ocean-node/commit/a3706e6ab94d38fde78a4fc04dc63afd0f68eefd) #### [v1.0.6](https://github.com/oceanprotocol/ocean-node/compare/v1.0.5...v1.0.6) @@ -235,28 +266,28 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). > 24 March 2025 -- Update node script - C2D [`#896`](https://github.com/oceanprotocol/ocean-node/pull/896) +- Update node script - C2D [`#896`](https://github.com/oceanprotocol/ocean-node/pull/896) - fix docker-compose [`#895`](https://github.com/oceanprotocol/ocean-node/pull/895) - re-indexing old DDOs [`#867`](https://github.com/oceanprotocol/ocean-node/pull/867) - Upgrade tsx dep to v4.x [`#893`](https://github.com/oceanprotocol/ocean-node/pull/893) - C2D Docker [`#705`](https://github.com/oceanprotocol/ocean-node/pull/705) - Updating codeowners [`#887`](https://github.com/oceanprotocol/ocean-node/pull/887) -- fix issue with empty nft fields [`#886`](https://github.com/oceanprotocol/ocean-node/pull/886) +- fix issue with empty nft fields [`#886`](https://github.com/oceanprotocol/ocean-node/pull/886) - add allowed admins access list [`#841`](https://github.com/oceanprotocol/ocean-node/pull/841) - Update error message for invalid peer connection [`#874`](https://github.com/oceanprotocol/ocean-node/pull/874) - add AUTHORIZED_DECRYPTERS_LIST [`#836`](https://github.com/oceanprotocol/ocean-node/pull/836) - fix status code if policy server not available [`#869`](https://github.com/oceanprotocol/ocean-node/pull/869) -- Fix DDO: Stats and Prices for exchanges/dispensers [`#774`](https://github.com/oceanprotocol/ocean-node/pull/774) +- Fix DDO: Stats and Prices for exchanges/dispensers [`#774`](https://github.com/oceanprotocol/ocean-node/pull/774) - move p2p getters as handlers [`#862`](https://github.com/oceanprotocol/ocean-node/pull/862) - always check remote peerId [`#864`](https://github.com/oceanprotocol/ocean-node/pull/864) - Test if dashboard changes are already committed [`#842`](https://github.com/oceanprotocol/ocean-node/pull/842) -- add AUTHORIZED*PUBLISHERS*\*\*\* env variables [`#826`](https://github.com/oceanprotocol/ocean-node/pull/826) +- add AUTHORIZED_PUBLISHERS_*** env variables [`#826`](https://github.com/oceanprotocol/ocean-node/pull/826) - Issue 814 credentials types [`#823`](https://github.com/oceanprotocol/ocean-node/pull/823) - remove echo command [`#839`](https://github.com/oceanprotocol/ocean-node/pull/839) - Issue 808 - new accesslist credentials type [`#819`](https://github.com/oceanprotocol/ocean-node/pull/819) - add ALLOWED_VALIDATORS_LIST [`#829`](https://github.com/oceanprotocol/ocean-node/pull/829) - update build files and hash [`#821`](https://github.com/oceanprotocol/ocean-node/pull/821) -- add \* as match all rule for address types [`#837`](https://github.com/oceanprotocol/ocean-node/pull/837) +- add * as match all rule for address types [`#837`](https://github.com/oceanprotocol/ocean-node/pull/837) - fix: return correct message on policy server call [`#834`](https://github.com/oceanprotocol/ocean-node/pull/834) - add policyServerPassthrough routes [`#832`](https://github.com/oceanprotocol/ocean-node/pull/832) - Bump fast-xml-parser from 4.3.6 to 4.5.0 in /dashboard [`#711`](https://github.com/oceanprotocol/ocean-node/pull/711) @@ -292,7 +323,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - fix system tests. Running old version of node? [`#733`](https://github.com/oceanprotocol/ocean-node/pull/733) - rm console.logs [`#731`](https://github.com/oceanprotocol/ocean-node/pull/731) - fix wrong block for log [`#727`](https://github.com/oceanprotocol/ocean-node/pull/727) -- nonce db sql lite [`#723`](https://github.com/oceanprotocol/ocean-node/pull/723) +- nonce db sql lite [`#723`](https://github.com/oceanprotocol/ocean-node/pull/723) - Bump version axios 1.6.0 -> 1.7.4. [`#716`](https://github.com/oceanprotocol/ocean-node/pull/716) - Bump version express 4.18.2 -> 4.21.0. [`#717`](https://github.com/oceanprotocol/ocean-node/pull/717) - Feature/ add Elasticsearch database alternative for typesense [`#599`](https://github.com/oceanprotocol/ocean-node/pull/599) @@ -301,7 +332,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - Bump micromatch from 4.0.5 to 4.0.8 in /dashboard [`#649`](https://github.com/oceanprotocol/ocean-node/pull/649) - Bump undici from 5.27.0 to 5.28.4 [`#610`](https://github.com/oceanprotocol/ocean-node/pull/610) - testing changes [`#718`](https://github.com/oceanprotocol/ocean-node/pull/718) -- Policy Server [`#694`](https://github.com/oceanprotocol/ocean-node/pull/694) +- Policy Server [`#694`](https://github.com/oceanprotocol/ocean-node/pull/694) - fix missing/invalid db_url log message, put warn at startup [`#654`](https://github.com/oceanprotocol/ocean-node/pull/654) - move c2d engines under OceanNode class [`#702`](https://github.com/oceanprotocol/ocean-node/pull/702) - improve error message, transfer fees and tweak node response [`#701`](https://github.com/oceanprotocol/ocean-node/pull/701) @@ -431,7 +462,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - Differentiate error messages indexer [`#570`](https://github.com/oceanprotocol/ocean-node/pull/570) - Issue 565 optimize get status [`#566`](https://github.com/oceanprotocol/ocean-node/pull/566) - fix get indexing queue [`#564`](https://github.com/oceanprotocol/ocean-node/pull/564) -- Changes on logging transports (.env var based locations) [`#553`](https://github.com/oceanprotocol/ocean-node/pull/553) +- Changes on logging transports (.env var based locations) [`#553`](https://github.com/oceanprotocol/ocean-node/pull/553) - Check if ddo state is active before executing node's commands. [`#542`](https://github.com/oceanprotocol/ocean-node/pull/542) - use static rpc provider [`#548`](https://github.com/oceanprotocol/ocean-node/pull/548) - Fix downloading full content of the file. [`#559`](https://github.com/oceanprotocol/ocean-node/pull/559) @@ -485,7 +516,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - Remove chain id from get compute envs task. [`#460`](https://github.com/oceanprotocol/ocean-node/pull/460) - Issue 397 warn env db logs [`#457`](https://github.com/oceanprotocol/ocean-node/pull/457) - fix p2p peers [`#449`](https://github.com/oceanprotocol/ocean-node/pull/449) -- c2d v2 arhitecture [`#381`](https://github.com/oceanprotocol/ocean-node/pull/381) +- c2d v2 arhitecture [`#381`](https://github.com/oceanprotocol/ocean-node/pull/381) - Fix: dashboard failing build if NODE_ENV is changed [`#450`](https://github.com/oceanprotocol/ocean-node/pull/450) - Dashboard: get ocean peers polling [`#445`](https://github.com/oceanprotocol/ocean-node/pull/445) - Replace hardcoded values in Dashboard [`#444`](https://github.com/oceanprotocol/ocean-node/pull/444) @@ -564,7 +595,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - Provider fees compute [`#252`](https://github.com/oceanprotocol/ocean-node/pull/252) - Updates to package.json bringing it in line with our other repositories [`#260`](https://github.com/oceanprotocol/ocean-node/pull/260) - Issue 205 ddo handling [`#239`](https://github.com/oceanprotocol/ocean-node/pull/239) -- fix error on publish + no signer/no account / metadata events error [`#255`](https://github.com/oceanprotocol/ocean-node/pull/255) +- fix error on publish + no signer/no account / metadata events error [`#255`](https://github.com/oceanprotocol/ocean-node/pull/255) - add config option for network interfaces, p2p and http [`#248`](https://github.com/oceanprotocol/ocean-node/pull/248) - Feature/ Add handle decrypt method [`#221`](https://github.com/oceanprotocol/ocean-node/pull/221) - Added checks for metadata events. [`#237`](https://github.com/oceanprotocol/ocean-node/pull/237) @@ -572,7 +603,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - Move commands from constants.ts to @types/commands.ts [`#244`](https://github.com/oceanprotocol/ocean-node/pull/244) - Issue 227 get environments [`#238`](https://github.com/oceanprotocol/ocean-node/pull/238) - fix unit test on commands.ts - pick mismatches in both directions [`#246`](https://github.com/oceanprotocol/ocean-node/pull/246) -- Expose validateDDO on http [`#234`](https://github.com/oceanprotocol/ocean-node/pull/234) +- Expose validateDDO on http [`#234`](https://github.com/oceanprotocol/ocean-node/pull/234) - Missing param validation for directCommand DOWNLOAD [`#242`](https://github.com/oceanprotocol/ocean-node/pull/242) - add c2d in ci [`#241`](https://github.com/oceanprotocol/ocean-node/pull/241) - add C2C cluster env config [`#240`](https://github.com/oceanprotocol/ocean-node/pull/240) diff --git a/docs/API.md b/docs/API.md index 862927df5..b141a61d9 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1463,6 +1463,7 @@ starts a free compute job and returns jobId if succesfull | additionalViewers | object | | optional array of addresses that are allowed to fetch the result | | queueMaxWaitTime | number | | optional max time in seconds a job can wait in the queue before being started | | encryptedDockerRegistryAuth | string | | Ecies encrypted docker auth schema for image (see [Private Docker Registries with Per-Job Authentication](../env.md#private-docker-registries-with-per-job-authentication)) | +| output | string | | Ecies encrypted with instructions for uploading compute results (see [C2D result upload to remote storage](../Storage.md#c2d-result-upload-to-remote-storage)) | #### Request diff --git a/docs/GPU.md b/docs/GPU.md index bbf36d210..b89a7664f 100644 --- a/docs/GPU.md +++ b/docs/GPU.md @@ -37,9 +37,9 @@ Fri Apr 25 06:00:34 2025 Now, time to get the id of the gpu: ```bash -root@gpu-1:/repos/ocean/ocean-node# nvidia-smi --query-gpu=name,uuid --format=csv -name, uuid -NVIDIA GeForce GTX 1060 3GB, GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81 +root@gpu-1:/repos/ocean/ocean-node# nvidia-smi --query-gpu=name,uuid,driver_version,memory.total --format=csv +name, uuid, driver version, memory total +NVIDIA GeForce GTX 1060 3GB, GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81, 570.195.03, 3072 MiB ``` Now, we can define the gpu for node: @@ -56,7 +56,9 @@ Now, we can define the gpu for node: "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], "Capabilities": [["gpu"]] } - } + }, + "driverVersion": "570.195.03", + "memoryTotal": "3072 MiB" } ``` @@ -80,7 +82,9 @@ Here is the full definition of DOCKER_COMPUTE_ENVIRONMENTS: "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], "Capabilities": [["gpu"]] } - } + }, + "driverVersion": "570.195.03", + "memoryTotal": "3072 MiB" }, { "id": "disk", "total": 1 } ], @@ -93,7 +97,7 @@ Here is the full definition of DOCKER_COMPUTE_ENVIRONMENTS: "feeToken": "0x123", "prices": [ { "id": "cpu", "price": 1 }, - { "id": "nyGPU", "price": 3 } + { "id": "myGPU", "price": 3 } ] } ] @@ -161,6 +165,8 @@ root@gpu-1:/repos/ocean/ocean-node# curl http://localhost:8000/api/services/comp "Capabilities": [["gpu"]] } }, + "driverVersion": "570.195.03", + "memoryTotal": "3072 MiB", "max": 1, "min": 0, "inUse": 0 @@ -259,7 +265,9 @@ Then define DOCKER_COMPUTE_ENVIRONMENTS with "seccomp": "unconfined" } } - } + }, + "driverVersion": "26.2.2", + "memoryTotal": "16384 MiB" }, { "id": "disk", @@ -316,8 +324,7 @@ Then define DOCKER_COMPUTE_ENVIRONMENTS with aka ```bash -export DOCKER_COMPUTE_ENVIRONMENTS="[{\"socketPath\":\"/var/run/docker.sock\",\"resources\":[{\"id\":\"myGPU\",\"description\":\"AMD Radeon RX 9070 XT\",\"type\":\"gpu\",\"total\":1,\"init\":{\"advanced\":{ -\"IpcMode\":\"host\",\"CapAdd\":[\"CAP_SYS_PTRACE\"],\"Devices\":[\"/dev/dxg\",\"/dev/dri/card0\"],\"Binds\":[\"/usr/lib/wsl/lib/libdxcore.so:/usr/lib/libdxcore.so\",\"/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1\"],\"SecurityOpt\":{\"seccomp\":\"unconfined\"}}}},{\"id\":\"disk\",\"total\":10}],\"storageExpiry\":604800,\"maxJobDuration\":3600,\"minJobDuration\":60,\"fees\":{\"1\":[{\"feeToken\":\"0x123\",\"prices\":[{\"id\":\"cpu\",\"price\":1},{\"id\":\"nyGPU\",\"price\":3}]}]},\"free\":{\"maxJobDuration\":60,\"minJobDuration\":10,\"maxJobs\":3,\"resources\":[{\"id\":\"cpu\",\"max\":1},{\"id\":\"ram\",\"max\":1},{\"id\":\"disk\",\"max\":1},{\"id\":\"myGPU\",\"max\":1}]}}]" +export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock","resources":[{"id":"myGPU","description":"AMD Radeon RX 9070 XT","type":"gpu","total":1,"init":{"advanced":{"IpcMode":"host","ShmSize":8589934592,"CapAdd":["SYS_PTRACE"],"Devices":["/dev/dxg","/dev/dri/card0"],"Binds":["/usr/lib/wsl/lib/libdxcore.so:/usr/lib/libdxcore.so","/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1"],"SecurityOpt":{"seccomp":"unconfined"}}},"driverVersion":"26.2.2","memoryTotal":"16384 MiB"},{"id":"disk","total":1}],"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"fees":{"1":[{"feeToken":"0x123","prices":[{"id":"cpu","price":1},{"id":"nyGPU","price":3}]}]},"free":{"maxJobDuration":60,"minJobDuration":10,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1},{"id":"myGPU","max":1}]}}]' ``` you should have it in your compute envs: @@ -390,6 +397,8 @@ root@gpu-1:/repos/ocean/ocean-node# curl http://localhost:8000/api/services/comp } } }, + "driverVersion": "26.2.2", + "memoryTotal": "16384 MiB", "max": 1, "min": 0, "inUse": 0 @@ -541,7 +550,9 @@ Now, we can define the GPU for the node: "GroupAdd": ["video", "render"], "CapAdd": ["SYS_ADMIN"] } - } + }, + "driverVersion": "32.0.101.8531", + "memoryTotal": "16384 MiB" } ``` @@ -563,7 +574,9 @@ Here is the full definition of DOCKER_COMPUTE_ENVIRONMENTS with Intel GPU: "GroupAdd": ["video", "render"], "CapAdd": ["SYS_ADMIN"] } - } + }, + "driverVersion": "32.0.101.8531", + "memoryTotal": "16384 MiB" }, { "id": "disk", "total": 1 } ], @@ -644,6 +657,8 @@ root@gpu-1:/repos/ocean/ocean-node# curl http://localhost:8000/api/services/comp "CapAdd": ["SYS_ADMIN"] } }, + "driverVersion": "32.0.101.8531", + "memoryTotal": "16384 MiB" "max": 1, "min": 0, "inUse": 0 diff --git a/docs/Publishing.md b/docs/Publishing.md new file mode 100644 index 000000000..ce8ecd0a0 --- /dev/null +++ b/docs/Publishing.md @@ -0,0 +1,158 @@ +# Publishing assets and isolated markets + +This document describes how assets (DDOs) move from a publisher’s machine into the Ocean network, how Ocean Nodes index them, and how you can restrict **who indexes what** to build **isolated markets**—logical subsets of the public network that share the same chain but apply stricter rules on the node side. + +--- + +## The default (public) flow + +In the usual setup, every Ocean Node that follows the network is free to index every asset that appears on chain, subject to normal validation (factory deployment, hashes, optional Policy Server hooks on the indexer, etc.). + +1. **Build the DDO locally** + You construct a valid DDO with [ocean.js](https://github.com/oceanprotocol/ocean.js) (or equivalent tooling). + +2. **Optional: validate against one or more nodes** + Before publishing, you can call the **`validateDDO`** command on a node (HTTP `POST /directCommand` or P2P). The node checks the DDO shape and rules; if [Policy Server](./PolicyServer.md) is configured, it can run extra authorization checks. On success, the node may return a **validation signature** payload (see [API](./API.md#validate-ddo)) used when you publish so the chain can record **which validator** approved the metadata. + +3. **Publish on chain** + You submit the metadata transaction (create/update) so the asset is anchored on the target chain. + +4. **Indexing** + Every node that indexes that chain processes the relevant events and stores the DDO in its local index (e.g. Typesense/Elasticsearch), so the asset becomes discoverable through those nodes’ APIs. + +Nothing in this flow *requires* an allowlist: if your nodes do not set the isolation options below, they behave as part of the open network. + +```mermaid +flowchart LR + A[Build DDO] --> B[Optional validateDDO] + B --> C[Publish on chain] + C --> D[Nodes index events] + D --> E[Query / consume via nodes] +``` + +--- + +## Isolated markets + +An **isolated market** is not a separate blockchain; it is a **policy** you enforce by configuring groups of Ocean Nodes so they only index (and thus only advertise) assets that satisfy your rules. You can combine several mechanisms. + +| Mechanism | What it restricts | Typical use | +|-----------|-------------------|-------------| +| `ALLOWED_VALIDATORS` / `ALLOWED_VALIDATORS_LIST` | Only assets whose publishing tx includes approval from **allowed validator addresses** | Org-approved catalog | +| `AUTHORIZED_PUBLISHERS` / `AUTHORIZED_PUBLISHERS_LIST` | Only assets whose **on-chain owner** is allowlisted | Known publishers only | +| `AUTHORIZED_DECRYPTERS` / `AUTHORIZED_DECRYPTERS_LIST` | Which **nodes** may call **`decryptDDO`** on the node that encrypted metadata | Encrypted DDO + indexer allowlist | + +Environment variable names and JSON shapes are documented in [env.md](./env.md). + +--- + +## Scenario 1 — Validator-gated indexing (`ALLOWED_VALIDATORS`) + +**Goal:** Only index assets that were **validated by a specific node** (or set of nodes), e.g. an organization that runs a “gatekeeper” node with extra checks. + +**How it works** + +1. **Gatekeeper node** + Run a node (`node1`) that performs `validateDDO`. Optionally connect it to a **Policy Server** so enterprise rules (SSO, LDAP, VC checks, etc.) run before the node signs—see [PolicyServer.md](./PolicyServer.md). + +2. **Validation signature** + When validation succeeds, the node returns signing material (the node signs a hash derived from the DDO using its **provider wallet**). Your client must pass this through **ocean.js** when publishing so the protocol emits **`MetadataValidated`** in the **same transaction** as metadata creation/update. Indexers look for those events and read **validator addresses** from them. + +3. **Isolated indexers** + On every node that should **only** follow that gatekeeper, set: + + ```bash + ALLOWED_VALIDATORS='["0xGatekeeperNodeProviderAddress"]' + ``` + + Use the **Ethereum address** of the gatekeeper node’s signing key (the same key used when producing the validation signature). You can list multiple addresses. + +4. **Access lists (optional)** + `ALLOWED_VALIDATORS_LIST` maps **chain ID → AccessList contract addresses**. If set, at least one on-chain validator must appear on those lists (see implementation and [env.md](./env.md)). + +**Outcome:** Nodes with `ALLOWED_VALIDATORS` set **skip** metadata events whose transaction does not include a `MetadataValidated` proof, or where none of the validators match your allowlist. Other nodes on the network—without this setting—still index everything as usual. + +--- + +## Scenario 2 — Publisher allowlists (`AUTHORIZED_PUBLISHERS`) + +**Goal:** Index only assets published by **specific wallets**, regardless of which validator signed (or use this together with Scenario 1). + +**How it works** + +- The indexer checks the **Data NFT owner** (`owner` from the metadata event) against your configuration. +- If `AUTHORIZED_PUBLISHERS` is non-empty, the owner must correspond to **one of the listed addresses** (compared case-insensitively against the configured list). +- If `AUTHORIZED_PUBLISHERS_LIST` is set, the owner must satisfy the **AccessList** contracts for that chain. + +**Example (conceptual)** + +```bash +# Only these publishers’ assets are indexed +AUTHORIZED_PUBLISHERS='["0xPublisherA...","0xPublisherB..."]' +``` + +**Combining with validators:** Set both `ALLOWED_VALIDATORS` and `AUTHORIZED_PUBLISHERS`. An asset is indexed only if it passes **both** checks. + +--- + +## Scenario 3 — Encrypted DDO and decrypt allowlists (`AUTHORIZED_DECRYPTERS`) + +**Goal:** At publish time, pick **one node** to encrypt the DDO; only **authorized** other nodes can decrypt it during indexing, so only those nodes can build a full index entry. + +**How it works** + +1. **Publishing** + When encrypting metadata for chain storage, you choose a node (often via its HTTP URL or peer id, depending on client flow). That node’s **`encrypt`** path prepares ciphertext that other parties cannot read without going through **`decryptDDO`**. + +2. **On-chain reference** + The encrypted payload is stored/ referenced such that indexers know **which node** can decrypt (HTTP URL of the encrypting node, or the node’s peer id for local decrypt). + +3. **Indexing and `decryptDDO`** + When another node indexes the asset, it must call the encrypting node’s **`POST /api/services/decrypt`** (with a nonce and signature proving the caller). The decrypting node checks whether the requester’s **`decrypterAddress`** (the indexing node’s own Ethereum address) is allowed. + +4. **`AUTHORIZED_DECRYPTERS` on the encrypting node** + On the node that holds the decryption keys, set: + + ```bash + AUTHORIZED_DECRYPTERS='["0xIndexerNode1...","0xIndexerNode2..."]' + ``` + + If this list is **non-empty**, only those addresses **or** the encrypting node itself may decrypt. Everyone else receives **403** and **cannot** complete indexing for that ciphertext. + +5. **Access lists (optional)** + `AUTHORIZED_DECRYPTERS_LIST` restricts callers via **AccessList** contracts per chain (see [env.md](./env.md)). + +**Outcome:** The asset may exist on chain for everyone, but **only nodes you list** can successfully decrypt and index it. Others fail at decrypt and leave the asset out of their index. + +--- + +## Combining scenarios (example) + +A private catalog might use: + +- **`ALLOWED_VALIDATORS`:** only your org’s gatekeeper node address. +- **`AUTHORIZED_PUBLISHERS`:** only approved data-owner wallets. +- **Encrypted DDO + `AUTHORIZED_DECRYPTERS`:** only your federation’s indexer node addresses can call `decrypt` on the encryption node. + +Tune each layer to match how much you trust **validators**, **publishers**, and **indexer machines**. + +--- + +## Other useful knobs + +- **`INDEXER_NETWORKS`:** Limit which chains a node indexes (see [env.md](./env.md)). +- **Policy Server on the indexer:** Even in public mode, your indexer can call Policy Server on **`newDDO`** / **`updateDDO`** to reject indexing—orthogonal to the allowlists above but complementary for org-wide policy. +- **`VALIDATE_UNSIGNED_DDO`:** Controls whether `validateDDO` requires publisher proofs before signing; relevant when hardening validation flows (see [env.md](./env.md)). + +--- + +## Where to read next + +| Topic | Doc | +|--------|-----| +| All env vars (`ALLOWED_*`, `AUTHORIZED_*`, lists) | [env.md](./env.md) | +| Policy Server actions (`validateDDO`, `newDDO`, …) | [PolicyServer.md](./PolicyServer.md) | +| `validateDDO` command shape | [API.md](./API.md#validate-ddo) | +| Key / signing model | [KeyManager.md](./KeyManager.md) | + +For client-side steps (exact ocean.js calls to pass validation into `MetadataValidated` and to choose an encrypting node), refer to the **ocean.js** documentation and examples for your stack version; the on-chain requirement from the node’s perspective is **`MetadataValidated` in the publishing transaction** when using `ALLOWED_VALIDATORS`, and successful **`/api/services/decrypt`** when metadata is encrypted. diff --git a/docs/Storage.md b/docs/Storage.md index 40f37e812..24c7ba7bf 100644 --- a/docs/Storage.md +++ b/docs/Storage.md @@ -1,15 +1,16 @@ # Storage Types -Ocean Node supports four storage backends for assets (e.g. algorithm or data files). Each type is identified by a `type` field on the file object and has its own shape and validation rules. +Ocean Node supports five storage backends for assets (e.g. algorithm or data files). Each type is identified by a `type` field on the file object and has its own shape and validation rules. ## Supported types -| Type | `type` value | Description | -| ---------- | ------------- | ------------------------------------ | -| **URL** | `url` | File served via HTTP/HTTPS | -| **IPFS** | `ipfs` | File identified by IPFS CID | -| **Arweave**| `arweave` | File identified by Arweave transaction ID | -| **S3** | `s3` | File in S3-compatible storage (AWS, Ceph, MinIO, etc.) | +| Type | `type` value | Description | +| ----------- | ------------ | ------------------------------------------------------ | +| **URL** | `url` | File served via HTTP/HTTPS | +| **IPFS** | `ipfs` | File identified by IPFS CID | +| **Arweave** | `arweave` | File identified by Arweave transaction ID | +| **S3** | `s3` | File in S3-compatible storage (AWS, Ceph, MinIO, etc.) | +| **FTP** | `ftp` | File served via FTP or FTPS | All file objects can optionally include encryption metadata: `encryptedBy` and `encryptMethod` (e.g. `AES`, `ECIES`). @@ -30,12 +31,12 @@ Files are fetched from a given URL using HTTP GET or POST. } ``` -| Field | Required | Description | -| --------- | -------- | ------------------------------------------------ | -| `type` | Yes | Must be `"url"` | -| `url` | Yes | Full HTTP/HTTPS URL to the file | -| `method` | Yes | `"get"` or `"post"` | -| `headers` | No | Optional request headers (key-value object) | +| Field | Required | Description | +| --------- | -------- | ------------------------------------------- | +| `type` | Yes | Must be `"url"` | +| `url` | Yes | Full HTTP/HTTPS URL to the file | +| `method` | Yes | `"get"` or `"post"` | +| `headers` | No | Optional request headers (key-value object) | ### Validation @@ -63,10 +64,10 @@ Files are resolved via an IPFS gateway using a content identifier (CID). } ``` -| Field | Required | Description | -| ------ | -------- | ------------------------------ | -| `type` | Yes | Must be `"ipfs"` | -| `hash` | Yes | IPFS content identifier (CID) | +| Field | Required | Description | +| ------ | -------- | ----------------------------- | +| `type` | Yes | Must be `"ipfs"` | +| `hash` | Yes | IPFS content identifier (CID) | The node builds the download URL as: `{ipfsGateway}/ipfs/{hash}` (e.g. `https://ipfs.io/ipfs/QmXoy...`). @@ -95,10 +96,10 @@ Files are identified by an Arweave transaction ID and fetched via an Arweave gat } ``` -| Field | Required | Description | -| --------------- | -------- | -------------------------- | -| `type` | Yes | Must be `"arweave"` | -| `transactionId` | Yes | Arweave transaction ID | +| Field | Required | Description | +| --------------- | -------- | ---------------------- | +| `type` | Yes | Must be `"arweave"` | +| `transactionId` | Yes | Arweave transaction ID | The node builds the download URL as: `{arweaveGateway}/{transactionId}`. @@ -134,21 +135,21 @@ Files are stored in S3-compatible object storage. The node uses the AWS SDK and } ``` -| Field | Required | Description | -| --------- | -------- | ----------- | -| `type` | Yes | Must be `"s3"` | -| `s3Access` | Yes | Object with endpoint, bucket, object key, and credentials (see below). | +| Field | Required | Description | +| ---------- | -------- | ---------------------------------------------------------------------- | +| `type` | Yes | Must be `"s3"` | +| `s3Access` | Yes | Object with endpoint, bucket, object key, and credentials (see below). | **`s3Access` fields:** -| Field | Required | Description | -| ----------------- | -------- | ----------- | -| `endpoint` | Yes | S3 endpoint URL (e.g. `https://s3.amazonaws.com`, `https://nyc3.digitaloceanspaces.com`, or `https://my-ceph.example.com`) | -| `bucket` | Yes | Bucket name | -| `objectKey` | Yes | Object key (path within the bucket) | -| `accessKeyId` | Yes | Access key for the S3-compatible API | -| `secretAccessKey` | Yes | Secret key for the S3-compatible API | -| `region` | No | Region (e.g. `us-east-1`). Optional; defaults to `us-east-1` if omitted. Some backends (e.g. Ceph) may ignore it. | +| Field | Required | Description | +| ----------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `endpoint` | Yes | S3 endpoint URL (e.g. `https://s3.amazonaws.com`, `https://nyc3.digitaloceanspaces.com`, or `https://my-ceph.example.com`) | +| `bucket` | Yes | Bucket name | +| `objectKey` | Yes | Object key (path within the bucket) | +| `accessKeyId` | Yes | Access key for the S3-compatible API | +| `secretAccessKey` | Yes | Secret key for the S3-compatible API | +| `region` | No | Region (e.g. `us-east-1`). Optional; defaults to `us-east-1` if omitted. Some backends (e.g. Ceph) may ignore it. | | `forcePathStyle` | No | If `true`, use path-style addressing (e.g. `endpoint/bucket/key`). Required for some S3-compatible services (e.g. MinIO). Default `false` (virtual-host style, e.g. `bucket.endpoint/key`, standard for AWS S3). | ### Validation @@ -163,11 +164,162 @@ Files are stored in S3-compatible object storage. The node uses the AWS SDK and --- +## FTP storage + +Files are fetched or uploaded via FTP or FTPS. The node uses a single `url` field containing the full FTP(S) URL (including optional credentials). Functionality mirrors URL storage: stream download, file metadata (size; content-type is `application/octet-stream`), and upload via STOR. + +### File object shape + +```json +{ + "type": "ftp", + "url": "ftp://user:password@ftp.example.com:21/path/to/file.zip" +} +``` + +For FTPS (TLS): + +```json +{ + "type": "ftp", + "url": "ftps://user:password@secure.example.com:990/pub/data.csv" +} +``` + +| Field | Required | Description | +| ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `type` | Yes | Must be `"ftp"` | +| `url` | Yes | Full FTP or FTPS URL. Supports `ftp://` and `ftps://`. May include credentials as `ftp://user:password@host:port/path`. Default port is 21 for FTP and 990 for FTPS. | + +### Validation + +- `url` must be present. +- URL must use protocol `ftp://` or `ftps://`. +- If the node config defines `unsafeURLs` (list of regex patterns), any URL matching a pattern is rejected. + +### Node configuration + +- Optional: `unsafeURLs` – array of regex strings; URLs matching any of them are considered unsafe and rejected (same as URL storage). + +### Upload + +FTPStorage supports `upload(filename, stream)`. If the file object’s `url` ends with `/`, the filename is appended to form the remote path; otherwise the URL is used as the full target path. Uses FTP STOR command. + +--- + +## C2D result upload to remote storage + +Compute-to-Data jobs can upload their output archive to a remote backend instead of keeping it only on local node disk. + +### How it works + +1. You build a `ComputeOutput` JSON object with: + - `remoteStorage`: one of the storage objects from this document (`url`, `s3`, `ftp`, etc.) + - optional `encryption`: currently only `AES` is accepted, with a hex key +2. You ECIES-encrypt that JSON into a string and send it in the compute command as `output`. +3. When the job finishes: + - if `output` is present and remote storage supports upload, Ocean Node uploads the tar archive remotely + - otherwise, Ocean Node falls back to local `outputs.tar` behavior + +### `ComputeOutput` shape + +```json +{ + "remoteStorage": { + "type": "s3", + "s3Access": { + "endpoint": "https://s3.amazonaws.com", + "region": "us-east-1", + "bucket": "my-c2d-results", + "objectKey": "jobs/result.tar", + "accessKeyId": "AKIA...", + "secretAccessKey": "..." + } + }, + "encryption": { + "encryptMethod": "AES", + "key": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } +} +``` + +Notes: + +- `output` itself is **not plain JSON** in the compute request; it must be an ECIES-encrypted string. +- `encryption.key` must be at least 32 bytes (64 hex chars). +- `encryption.encryptMethod` must be `AES` if provided. + +### End-to-end example + +#### 1) Create plaintext output instructions + +```json +{ + "remoteStorage": { + "type": "ftp", + "url": "ftp://user:password@ftp.example.com:21/results/" + }, + "encryption": { + "encryptMethod": "AES", + "key": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } +} +``` + +#### 2) Encrypt the JSON + +You can use `POST /api/services/encrypt` to encrypt the JSON string for Ocean Node: + +```bash +curl -X POST "https:///api/services/encrypt?consumerAddress=<0xAddress>&nonce=&signature=" \ + -H "Content-Type: text/plain" \ + --data-raw '{"remoteStorage":{"type":"ftp","url":"ftp://user:password@ftp.example.com:21/results/"},"encryption":{"encryptMethod":"AES","key":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}}' +``` + +The response is the encrypted blob (hex string). +If your encrypt response includes `0x` prefix, remove it before sending as compute `output` (compute handlers decode `output` as raw hex bytes). + +#### 3) Send compute command with `output` + +Example for `freeStartCompute`: + +```json +{ + "command": "freeStartCompute", + "consumerAddress": "0x...", + "signature": "0x...", + "nonce": "123", + "environment": "", + "datasets": [], + "algorithm": { + "meta": { + "rawcode": "print('hello')", + "container": { + "image": "python", + "tag": "3.10", + "entrypoint": "python", + "checksum": "..." + } + } + }, + "output": "" +} +``` + +### Uploaded filename and fallback behavior + +- For remote upload, Ocean Node writes: `outputs--.tar` +- If `output` is missing/empty, or chosen storage does not support upload, Ocean Node stores output locally (`outputs.tar`) as before. +- If remote upload fails, job status is set to `ResultsUploadFailed`. + +--- + ## Summary - **URL**: flexible HTTP(S) endpoints; optional custom headers and `unsafeURLs` filtering. - **IPFS**: CID-based; requires `ipfsGateway` in config. - **Arweave**: transaction-ID-based; requires `arweaveGateway` in config. - **S3**: S3-compatible object storage (AWS, Ceph, MinIO, etc.); credentials and endpoint in the file object; `region` optional (defaults to `us-east-1`). +- **FTP**: FTP/FTPS URLs; stream download, metadata (size), and upload via STOR; optional `unsafeURLs` filtering. The storage implementation lives under `src/components/storage/`. The node selects the backend from the file object’s `type` (case-insensitive) and validates the shape and config before fetching or streaming the file. diff --git a/docs/env.md b/docs/env.md index 4f2445f03..beeae0180 100644 --- a/docs/env.md +++ b/docs/env.md @@ -78,6 +78,20 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `ELASTICSEARCH_SNIFF_ON_CONNECTION_FAULT`: Enable automatic cluster node discovery when connection faults occur. Default is `true`. Example: `true` - `ELASTICSEARCH_HEALTH_CHECK_INTERVAL`: Interval in milliseconds for proactive connection health monitoring. Default is `60000`. Example: `60000` +## Database + +- `DB_URL`: URL for connecting to the database. Required for running a database with the node. Example: `"http://localhost:8108/?apiKey=xyz"` +- `DB_USERNAME`: Username for database authentication. Optional if not using authentication. Example: `"elastic"` +- `DB_PASSWORD`: Password for database authentication. Optional if not using authentication. Example: `"password123"` +- `ELASTICSEARCH_REQUEST_TIMEOUT`: Request timeout in milliseconds for Elasticsearch operations. Default is `60000`. Example: `60000` +- `ELASTICSEARCH_PING_TIMEOUT`: Ping timeout in milliseconds for Elasticsearch health checks. Default is `5000`. Example: `5000` +- `ELASTICSEARCH_RESURRECT_STRATEGY`: Strategy for bringing failed Elasticsearch nodes back online. Options are 'ping', 'optimistic', or 'none'. Default is `ping`. Example: `"ping"` +- `ELASTICSEARCH_MAX_RETRIES`: Maximum number of retry attempts for failed Elasticsearch operations. Default is `5`. Example: `5` +- `ELASTICSEARCH_SNIFF_ON_START`: Enable cluster node discovery on Elasticsearch client startup. Default is `true`. Example: `true` +- `ELASTICSEARCH_SNIFF_INTERVAL`: Interval in milliseconds for periodic cluster health monitoring and node discovery. Set to 'false' to disable. Default is `30000`. Example: `30000` +- `ELASTICSEARCH_SNIFF_ON_CONNECTION_FAULT`: Enable automatic cluster node discovery when connection faults occur. Default is `true`. Example: `true` +- `ELASTICSEARCH_HEALTH_CHECK_INTERVAL`: Interval in milliseconds for proactive connection health monitoring. Default is `60000`. Example: `60000` + ## Payments - `ESCROW_CLAIM_TIMEOUT`: Amount of time reserved to claim a escrow payment, in seconds. Defaults to `3600`. Example: `3600` diff --git a/package-lock.json b/package-lock.json index fc47b8c8f..94d1a8492 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,17 @@ { "name": "ocean-node", - "version": "1.0.7", + "version": "2.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ocean-node", - "version": "1.0.7", + "version": "2.0.2", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-s3": "^3.1002.0", + "@aws-sdk/lib-storage": "^3.1002.0", "@chainsafe/libp2p-noise": "^17.0.0", "@chainsafe/libp2p-yamux": "^8.0.1", "@elastic/elasticsearch": "^8.14.0", @@ -38,6 +39,7 @@ "@oceanprotocol/ddo-js": "^0.2.0", "axios": "^1.13.5", "base58-js": "^2.0.0", + "basic-ftp": "^5.2.0", "cors": "^2.8.5", "datastore-level": "^12.0.2", "delay": "^5.0.0", @@ -58,7 +60,7 @@ "node-cron": "^3.0.3", "sqlite3": "^5.1.7", "stream-concat": "^1.0.0", - "tar": "^7.5.10", + "tar": "^7.5.11", "uint8arrays": "^4.0.6", "url-join": "^5.0.0", "winston": "^3.11.0", @@ -354,65 +356,65 @@ } }, "node_modules/@aws-sdk/client-s3": { - "version": "3.1004.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1004.0.tgz", - "integrity": "sha512-m0zNfpsona9jQdX1cHtHArOiuvSGZPsgp/KRZS2YjJhKah96G2UN3UNGZQ6aVjXIQjCY6UanCJo0uW9Xf2U41w==", + "version": "3.1009.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1009.0.tgz", + "integrity": "sha512-luy8CxallkoiGWTqU86ca/BbvkWJjs0oala7uIIRN1JtQxMb5i4Yl/PBZVcQFhbK9kQi0PK0GfD8gIpLkI91fw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/credential-provider-node": "^3.972.18", - "@aws-sdk/middleware-bucket-endpoint": "^3.972.7", - "@aws-sdk/middleware-expect-continue": "^3.972.7", - "@aws-sdk/middleware-flexible-checksums": "^3.973.4", - "@aws-sdk/middleware-host-header": "^3.972.7", - "@aws-sdk/middleware-location-constraint": "^3.972.7", - "@aws-sdk/middleware-logger": "^3.972.7", - "@aws-sdk/middleware-recursion-detection": "^3.972.7", - "@aws-sdk/middleware-sdk-s3": "^3.972.18", - "@aws-sdk/middleware-ssec": "^3.972.7", - "@aws-sdk/middleware-user-agent": "^3.972.19", - "@aws-sdk/region-config-resolver": "^3.972.7", - "@aws-sdk/signature-v4-multi-region": "^3.996.6", - "@aws-sdk/types": "^3.973.5", - "@aws-sdk/util-endpoints": "^3.996.4", - "@aws-sdk/util-user-agent-browser": "^3.972.7", - "@aws-sdk/util-user-agent-node": "^3.973.4", - "@smithy/config-resolver": "^4.4.10", - "@smithy/core": "^3.23.8", - "@smithy/eventstream-serde-browser": "^4.2.11", - "@smithy/eventstream-serde-config-resolver": "^4.3.11", - "@smithy/eventstream-serde-node": "^4.2.11", - "@smithy/fetch-http-handler": "^5.3.13", - "@smithy/hash-blob-browser": "^4.2.12", - "@smithy/hash-node": "^4.2.11", - "@smithy/hash-stream-node": "^4.2.11", - "@smithy/invalid-dependency": "^4.2.11", - "@smithy/md5-js": "^4.2.11", - "@smithy/middleware-content-length": "^4.2.11", - "@smithy/middleware-endpoint": "^4.4.22", - "@smithy/middleware-retry": "^4.4.39", - "@smithy/middleware-serde": "^4.2.12", - "@smithy/middleware-stack": "^4.2.11", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/node-http-handler": "^4.4.14", - "@smithy/protocol-http": "^5.3.11", - "@smithy/smithy-client": "^4.12.2", - "@smithy/types": "^4.13.0", - "@smithy/url-parser": "^4.2.11", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/credential-provider-node": "^3.972.21", + "@aws-sdk/middleware-bucket-endpoint": "^3.972.8", + "@aws-sdk/middleware-expect-continue": "^3.972.8", + "@aws-sdk/middleware-flexible-checksums": "^3.973.6", + "@aws-sdk/middleware-host-header": "^3.972.8", + "@aws-sdk/middleware-location-constraint": "^3.972.8", + "@aws-sdk/middleware-logger": "^3.972.8", + "@aws-sdk/middleware-recursion-detection": "^3.972.8", + "@aws-sdk/middleware-sdk-s3": "^3.972.20", + "@aws-sdk/middleware-ssec": "^3.972.8", + "@aws-sdk/middleware-user-agent": "^3.972.21", + "@aws-sdk/region-config-resolver": "^3.972.8", + "@aws-sdk/signature-v4-multi-region": "^3.996.8", + "@aws-sdk/types": "^3.973.6", + "@aws-sdk/util-endpoints": "^3.996.5", + "@aws-sdk/util-user-agent-browser": "^3.972.8", + "@aws-sdk/util-user-agent-node": "^3.973.7", + "@smithy/config-resolver": "^4.4.11", + "@smithy/core": "^3.23.11", + "@smithy/eventstream-serde-browser": "^4.2.12", + "@smithy/eventstream-serde-config-resolver": "^4.3.12", + "@smithy/eventstream-serde-node": "^4.2.12", + "@smithy/fetch-http-handler": "^5.3.15", + "@smithy/hash-blob-browser": "^4.2.13", + "@smithy/hash-node": "^4.2.12", + "@smithy/hash-stream-node": "^4.2.12", + "@smithy/invalid-dependency": "^4.2.12", + "@smithy/md5-js": "^4.2.12", + "@smithy/middleware-content-length": "^4.2.12", + "@smithy/middleware-endpoint": "^4.4.25", + "@smithy/middleware-retry": "^4.4.42", + "@smithy/middleware-serde": "^4.2.14", + "@smithy/middleware-stack": "^4.2.12", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/node-http-handler": "^4.4.16", + "@smithy/protocol-http": "^5.3.12", + "@smithy/smithy-client": "^4.12.5", + "@smithy/types": "^4.13.1", + "@smithy/url-parser": "^4.2.12", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.38", - "@smithy/util-defaults-mode-node": "^4.2.41", - "@smithy/util-endpoints": "^3.3.2", - "@smithy/util-middleware": "^4.2.11", - "@smithy/util-retry": "^4.2.11", - "@smithy/util-stream": "^4.5.17", + "@smithy/util-defaults-mode-browser": "^4.3.41", + "@smithy/util-defaults-mode-node": "^4.2.44", + "@smithy/util-endpoints": "^3.3.3", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-retry": "^4.2.12", + "@smithy/util-stream": "^4.5.19", "@smithy/util-utf8": "^4.2.2", - "@smithy/util-waiter": "^4.2.11", + "@smithy/util-waiter": "^4.2.13", "tslib": "^2.6.2" }, "engines": { @@ -420,22 +422,22 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.973.18", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.18.tgz", - "integrity": "sha512-GUIlegfcK2LO1J2Y98sCJy63rQSiLiDOgVw7HiHPRqfI2vb3XozTVqemwO0VSGXp54ngCnAQz0Lf0YPCBINNxA==", + "version": "3.973.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.20.tgz", + "integrity": "sha512-i3GuX+lowD892F3IuJf8o6AbyDupMTdyTxQrCJGcn71ni5hTZ82L4nQhcdumxZ7XPJRJJVHS/CR3uYOIIs0PVA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", - "@aws-sdk/xml-builder": "^3.972.10", - "@smithy/core": "^3.23.8", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/property-provider": "^4.2.11", - "@smithy/protocol-http": "^5.3.11", - "@smithy/signature-v4": "^5.3.11", - "@smithy/smithy-client": "^4.12.2", - "@smithy/types": "^4.13.0", + "@aws-sdk/types": "^3.973.6", + "@aws-sdk/xml-builder": "^3.972.11", + "@smithy/core": "^3.23.11", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/property-provider": "^4.2.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/signature-v4": "^5.3.12", + "@smithy/smithy-client": "^4.12.5", + "@smithy/types": "^4.13.1", "@smithy/util-base64": "^4.3.2", - "@smithy/util-middleware": "^4.2.11", + "@smithy/util-middleware": "^4.2.12", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -444,12 +446,12 @@ } }, "node_modules/@aws-sdk/crc64-nvme": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.4.tgz", - "integrity": "sha512-HKZIZLbRyvzo/bXZU7Zmk6XqU+1C9DjI56xd02vwuDIxedxBEqP17t9ExhbP9QFeNq/a3l9GOcyirFXxmbDhmw==", + "version": "3.972.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.5.tgz", + "integrity": "sha512-2VbTstbjKdT+yKi8m7b3a9CiVac+pL/IY2PHJwsaGkkHmuuqkJZIErPck1h6P3T9ghQMLSdMPyW6Qp7Di5swFg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -457,15 +459,15 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.16", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.16.tgz", - "integrity": "sha512-HrdtnadvTGAQUr18sPzGlE5El3ICphnH6SU7UQOMOWFgRKbTRNN8msTxM4emzguUso9CzaHU2xy5ctSrmK5YNA==", + "version": "3.972.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.18.tgz", + "integrity": "sha512-X0B8AlQY507i5DwjLByeU2Af4ARsl9Vr84koDcXCbAkplmU+1xBFWxEPrWRAoh56waBne/yJqEloSwvRf4x6XA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/types": "^3.973.5", - "@smithy/property-provider": "^4.2.11", - "@smithy/types": "^4.13.0", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/types": "^3.973.6", + "@smithy/property-provider": "^4.2.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -473,20 +475,20 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.18", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.18.tgz", - "integrity": "sha512-NyB6smuZAixND5jZumkpkunQ0voc4Mwgkd+SZ6cvAzIB7gK8HV8Zd4rS8Kn5MmoGgusyNfVGG+RLoYc4yFiw+A==", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.20.tgz", + "integrity": "sha512-ey9Lelj001+oOfrbKmS6R2CJAiXX7QKY4Vj9VJv6L2eE6/VjD8DocHIoYqztTm70xDLR4E1jYPTKfIui+eRNDA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/types": "^3.973.5", - "@smithy/fetch-http-handler": "^5.3.13", - "@smithy/node-http-handler": "^4.4.14", - "@smithy/property-provider": "^4.2.11", - "@smithy/protocol-http": "^5.3.11", - "@smithy/smithy-client": "^4.12.2", - "@smithy/types": "^4.13.0", - "@smithy/util-stream": "^4.5.17", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/types": "^3.973.6", + "@smithy/fetch-http-handler": "^5.3.15", + "@smithy/node-http-handler": "^4.4.16", + "@smithy/property-provider": "^4.2.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/smithy-client": "^4.12.5", + "@smithy/types": "^4.13.1", + "@smithy/util-stream": "^4.5.19", "tslib": "^2.6.2" }, "engines": { @@ -494,24 +496,24 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.17.tgz", - "integrity": "sha512-dFqh7nfX43B8dO1aPQHOcjC0SnCJ83H3F+1LoCh3X1P7E7N09I+0/taID0asU6GCddfDExqnEvQtDdkuMe5tKQ==", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.20.tgz", + "integrity": "sha512-5flXSnKHMloObNF+9N0cupKegnH1Z37cdVlpETVgx8/rAhCe+VNlkcZH3HDg2SDn9bI765S+rhNPXGDJJPfbtA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/credential-provider-env": "^3.972.16", - "@aws-sdk/credential-provider-http": "^3.972.18", - "@aws-sdk/credential-provider-login": "^3.972.17", - "@aws-sdk/credential-provider-process": "^3.972.16", - "@aws-sdk/credential-provider-sso": "^3.972.17", - "@aws-sdk/credential-provider-web-identity": "^3.972.17", - "@aws-sdk/nested-clients": "^3.996.7", - "@aws-sdk/types": "^3.973.5", - "@smithy/credential-provider-imds": "^4.2.11", - "@smithy/property-provider": "^4.2.11", - "@smithy/shared-ini-file-loader": "^4.4.6", - "@smithy/types": "^4.13.0", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/credential-provider-env": "^3.972.18", + "@aws-sdk/credential-provider-http": "^3.972.20", + "@aws-sdk/credential-provider-login": "^3.972.20", + "@aws-sdk/credential-provider-process": "^3.972.18", + "@aws-sdk/credential-provider-sso": "^3.972.20", + "@aws-sdk/credential-provider-web-identity": "^3.972.20", + "@aws-sdk/nested-clients": "^3.996.10", + "@aws-sdk/types": "^3.973.6", + "@smithy/credential-provider-imds": "^4.2.12", + "@smithy/property-provider": "^4.2.12", + "@smithy/shared-ini-file-loader": "^4.4.7", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -519,18 +521,18 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.17.tgz", - "integrity": "sha512-gf2E5b7LpKb+JX2oQsRIDxdRZjBFZt2olCGlWCdb3vBERbXIPgm2t1R5mEnwd4j0UEO/Tbg5zN2KJbHXttJqwA==", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.20.tgz", + "integrity": "sha512-gEWo54nfqp2jABMu6HNsjVC4hDLpg9HC8IKSJnp0kqWtxIJYHTmiLSsIfI4ScQjxEwpB+jOOH8dOLax1+hy/Hw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/nested-clients": "^3.996.7", - "@aws-sdk/types": "^3.973.5", - "@smithy/property-provider": "^4.2.11", - "@smithy/protocol-http": "^5.3.11", - "@smithy/shared-ini-file-loader": "^4.4.6", - "@smithy/types": "^4.13.0", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/nested-clients": "^3.996.10", + "@aws-sdk/types": "^3.973.6", + "@smithy/property-provider": "^4.2.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/shared-ini-file-loader": "^4.4.7", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -538,22 +540,22 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.18", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.18.tgz", - "integrity": "sha512-ZDJa2gd1xiPg/nBDGhUlat02O8obaDEnICBAVS8qieZ0+nDfaB0Z3ec6gjZj27OqFTjnB/Q5a0GwQwb7rMVViw==", + "version": "3.972.21", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.21.tgz", + "integrity": "sha512-hah8if3/B/Q+LBYN5FukyQ1Mym6PLPDsBOBsIgNEYD6wLyZg0UmUF/OKIVC3nX9XH8TfTPuITK+7N/jenVACWA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.16", - "@aws-sdk/credential-provider-http": "^3.972.18", - "@aws-sdk/credential-provider-ini": "^3.972.17", - "@aws-sdk/credential-provider-process": "^3.972.16", - "@aws-sdk/credential-provider-sso": "^3.972.17", - "@aws-sdk/credential-provider-web-identity": "^3.972.17", - "@aws-sdk/types": "^3.973.5", - "@smithy/credential-provider-imds": "^4.2.11", - "@smithy/property-provider": "^4.2.11", - "@smithy/shared-ini-file-loader": "^4.4.6", - "@smithy/types": "^4.13.0", + "@aws-sdk/credential-provider-env": "^3.972.18", + "@aws-sdk/credential-provider-http": "^3.972.20", + "@aws-sdk/credential-provider-ini": "^3.972.20", + "@aws-sdk/credential-provider-process": "^3.972.18", + "@aws-sdk/credential-provider-sso": "^3.972.20", + "@aws-sdk/credential-provider-web-identity": "^3.972.20", + "@aws-sdk/types": "^3.973.6", + "@smithy/credential-provider-imds": "^4.2.12", + "@smithy/property-provider": "^4.2.12", + "@smithy/shared-ini-file-loader": "^4.4.7", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -561,16 +563,16 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.16", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.16.tgz", - "integrity": "sha512-n89ibATwnLEg0ZdZmUds5bq8AfBAdoYEDpqP3uzPLaRuGelsKlIvCYSNNvfgGLi8NaHPNNhs1HjJZYbqkW9b+g==", + "version": "3.972.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.18.tgz", + "integrity": "sha512-Tpl7SRaPoOLT32jbTWchPsn52hYYgJ0kpiFgnwk8pxTANQdUymVSZkzFvv1+oOgZm1CrbQUP9MBeoMZ9IzLZjA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/types": "^3.973.5", - "@smithy/property-provider": "^4.2.11", - "@smithy/shared-ini-file-loader": "^4.4.6", - "@smithy/types": "^4.13.0", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/types": "^3.973.6", + "@smithy/property-provider": "^4.2.12", + "@smithy/shared-ini-file-loader": "^4.4.7", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -578,18 +580,18 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.17.tgz", - "integrity": "sha512-wGtte+48xnhnhHMl/MsxzacBPs5A+7JJedjiP452IkHY7vsbYKcvQBqFye8LwdTJVeHtBHv+JFeTscnwepoWGg==", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.20.tgz", + "integrity": "sha512-p+R+PYR5Z7Gjqf/6pvbCnzEHcqPCpLzR7Yf127HjJ6EAb4hUcD+qsNRnuww1sB/RmSeCLxyay8FMyqREw4p1RA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/nested-clients": "^3.996.7", - "@aws-sdk/token-providers": "3.1004.0", - "@aws-sdk/types": "^3.973.5", - "@smithy/property-provider": "^4.2.11", - "@smithy/shared-ini-file-loader": "^4.4.6", - "@smithy/types": "^4.13.0", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/nested-clients": "^3.996.10", + "@aws-sdk/token-providers": "3.1009.0", + "@aws-sdk/types": "^3.973.6", + "@smithy/property-provider": "^4.2.12", + "@smithy/shared-ini-file-loader": "^4.4.7", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -597,34 +599,55 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.17.tgz", - "integrity": "sha512-8aiVJh6fTdl8gcyL+sVNcNwTtWpmoFa1Sh7xlj6Z7L/cZ/tYMEBHq44wTYG8Kt0z/PpGNopD89nbj3FHl9QmTA==", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.20.tgz", + "integrity": "sha512-rWCmh8o7QY4CsUj63qopzMzkDq/yPpkrpb+CnjBEFSOg/02T/we7sSTVg4QsDiVS9uwZ8VyONhq98qt+pIh3KA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/nested-clients": "^3.996.10", + "@aws-sdk/types": "^3.973.6", + "@smithy/property-provider": "^4.2.12", + "@smithy/shared-ini-file-loader": "^4.4.7", + "@smithy/types": "^4.13.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/lib-storage": { + "version": "3.1009.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.1009.0.tgz", + "integrity": "sha512-gHQh1sNeTuxZxPSMSQWOq/Xli8I5499uWyRKMakMSv8N7IYfoyDdyT52Ul6697qcqVaoPHixmYTllfEWMo1AKg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/nested-clients": "^3.996.7", - "@aws-sdk/types": "^3.973.5", - "@smithy/property-provider": "^4.2.11", - "@smithy/shared-ini-file-loader": "^4.4.6", - "@smithy/types": "^4.13.0", + "@smithy/abort-controller": "^4.2.12", + "@smithy/middleware-endpoint": "^4.4.25", + "@smithy/smithy-client": "^4.12.5", + "buffer": "5.6.0", + "events": "3.3.0", + "stream-browserify": "3.0.0", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.1009.0" } }, "node_modules/@aws-sdk/middleware-bucket-endpoint": { - "version": "3.972.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.7.tgz", - "integrity": "sha512-goX+axlJ6PQlRnzE2bQisZ8wVrlm6dXJfBzMJhd8LhAIBan/w1Kl73fJnalM/S+18VnpzIHumyV6DtgmvqG5IA==", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.8.tgz", + "integrity": "sha512-WR525Rr2QJSETa9a050isktyWi/4yIGcmY3BQ1kpHqb0LqUglQHCS8R27dTJxxWNZvQ0RVGtEZjTCbZJpyF3Aw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", + "@aws-sdk/types": "^3.973.6", "@aws-sdk/util-arn-parser": "^3.972.3", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", "@smithy/util-config-provider": "^4.2.2", "tslib": "^2.6.2" }, @@ -633,14 +656,14 @@ } }, "node_modules/@aws-sdk/middleware-expect-continue": { - "version": "3.972.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.7.tgz", - "integrity": "sha512-mvWqvm61bmZUKmmrtl2uWbokqpenY3Mc3Jf4nXB/Hse6gWxLPaCQThmhPBDzsPSV8/Odn8V6ovWt3pZ7vy4BFQ==", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.8.tgz", + "integrity": "sha512-5DTBTiotEES1e2jOHAq//zyzCjeMB78lEHd35u15qnrid4Nxm7diqIf9fQQ3Ov0ChH1V3Vvt13thOnrACmfGVQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", + "@aws-sdk/types": "^3.973.6", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -648,23 +671,23 @@ } }, "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.973.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.973.4.tgz", - "integrity": "sha512-7CH2jcGmkvkHc5Buz9IGbdjq1729AAlgYJiAvGq7qhCHqYleCsriWdSnmsqWTwdAfXHMT+pkxX3w6v5tJNcSug==", + "version": "3.973.6", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.973.6.tgz", + "integrity": "sha512-0nYEgkJH7Yt9k+nZJyllTghnkKaz17TWFcr5Mi0XMVMzYlF4ytDZADQpF2/iJo36cKL5AYSzRsvlykE4M/ErTA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/crc32": "5.2.0", "@aws-crypto/crc32c": "5.2.0", "@aws-crypto/util": "5.2.0", - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/crc64-nvme": "^3.972.4", - "@aws-sdk/types": "^3.973.5", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/crc64-nvme": "^3.972.5", + "@aws-sdk/types": "^3.973.6", "@smithy/is-array-buffer": "^4.2.2", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", - "@smithy/util-middleware": "^4.2.11", - "@smithy/util-stream": "^4.5.17", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-stream": "^4.5.19", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -673,14 +696,14 @@ } }, "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.972.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.7.tgz", - "integrity": "sha512-aHQZgztBFEpDU1BB00VWCIIm85JjGjQW1OG9+98BdmaOpguJvzmXBGbnAiYcciCd+IS4e9BEq664lhzGnWJHgQ==", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.8.tgz", + "integrity": "sha512-wAr2REfKsqoKQ+OkNqvOShnBoh+nkPurDKW7uAeVSu6kUECnWlSJiPvnoqxGlfousEY/v9LfS9sNc46hjSYDIQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", + "@aws-sdk/types": "^3.973.6", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -688,13 +711,13 @@ } }, "node_modules/@aws-sdk/middleware-location-constraint": { - "version": "3.972.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.7.tgz", - "integrity": "sha512-vdK1LJfffBp87Lj0Bw3WdK1rJk9OLDYdQpqoKgmpIZPe+4+HawZ6THTbvjhJt4C4MNnRrHTKHQjkwBiIpDBoig==", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.8.tgz", + "integrity": "sha512-KaUoFuoFPziIa98DSQsTPeke1gvGXlc5ZGMhy+b+nLxZ4A7jmJgLzjEF95l8aOQN2T/qlPP3MrAyELm8ExXucw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", - "@smithy/types": "^4.13.0", + "@aws-sdk/types": "^3.973.6", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -702,13 +725,13 @@ } }, "node_modules/@aws-sdk/middleware-logger": { - "version": "3.972.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.7.tgz", - "integrity": "sha512-LXhiWlWb26txCU1vcI9PneESSeRp/RYY/McuM4SpdrimQR5NgwaPb4VJCadVeuGWgh6QmqZ6rAKSoL1ob16W6w==", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.8.tgz", + "integrity": "sha512-CWl5UCM57WUFaFi5kB7IBY1UmOeLvNZAZ2/OZ5l20ldiJ3TiIz1pC65gYj8X0BCPWkeR1E32mpsCk1L1I4n+lA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", - "@smithy/types": "^4.13.0", + "@aws-sdk/types": "^3.973.6", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -716,15 +739,15 @@ } }, "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.972.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.7.tgz", - "integrity": "sha512-l2VQdcBcYLzIzykCHtXlbpiVCZ94/xniLIkAj0jpnpjY4xlgZx7f56Ypn+uV1y3gG0tNVytJqo3K9bfMFee7SQ==", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.8.tgz", + "integrity": "sha512-BnnvYs2ZEpdlmZ2PNlV2ZyQ8j8AEkMTjN79y/YA475ER1ByFYrkVR85qmhni8oeTaJcDqbx364wDpitDAA/wCA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", + "@aws-sdk/types": "^3.973.6", "@aws/lambda-invoke-store": "^0.2.2", - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -732,23 +755,23 @@ } }, "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.972.18", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.18.tgz", - "integrity": "sha512-5E3XxaElrdyk6ZJ0TjH7Qm6ios4b/qQCiLr6oQ8NK7e4Kn6JBTJCaYioQCQ65BpZ1+l1mK5wTAac2+pEz0Smpw==", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.20.tgz", + "integrity": "sha512-yhva/xL5H4tWQgsBjwV+RRD0ByCzg0TcByDCLp3GXdn/wlyRNfy8zsswDtCvr1WSKQkSQYlyEzPuWkJG0f5HvQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/types": "^3.973.5", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/types": "^3.973.6", "@aws-sdk/util-arn-parser": "^3.972.3", - "@smithy/core": "^3.23.8", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/protocol-http": "^5.3.11", - "@smithy/signature-v4": "^5.3.11", - "@smithy/smithy-client": "^4.12.2", - "@smithy/types": "^4.13.0", + "@smithy/core": "^3.23.11", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/signature-v4": "^5.3.12", + "@smithy/smithy-client": "^4.12.5", + "@smithy/types": "^4.13.1", "@smithy/util-config-provider": "^4.2.2", - "@smithy/util-middleware": "^4.2.11", - "@smithy/util-stream": "^4.5.17", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-stream": "^4.5.19", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -757,13 +780,13 @@ } }, "node_modules/@aws-sdk/middleware-ssec": { - "version": "3.972.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.7.tgz", - "integrity": "sha512-G9clGVuAml7d8DYzY6DnRi7TIIDRvZ3YpqJPz/8wnWS5fYx/FNWNmkO6iJVlVkQg9BfeMzd+bVPtPJOvC4B+nQ==", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.8.tgz", + "integrity": "sha512-wqlK0yO/TxEC2UsY9wIlqeeutF6jjLe0f96Pbm40XscTo57nImUk9lBcw0dPgsm0sppFtAkSlDrfpK+pC30Wqw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", - "@smithy/types": "^4.13.0", + "@aws-sdk/types": "^3.973.6", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -771,18 +794,18 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.972.19", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.19.tgz", - "integrity": "sha512-Km90fcXt3W/iqujHzuM6IaDkYCj73gsYufcuWXApWdzoTy6KGk8fnchAjePMARU0xegIR3K4N3yIo1vy7OVe8A==", + "version": "3.972.21", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.21.tgz", + "integrity": "sha512-62XRl1GDYPpkt7cx1AX1SPy9wgNE9Iw/NPuurJu4lmhCWS7sGKO+kS53TQ8eRmIxy3skmvNInnk0ZbWrU5Dpyg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/types": "^3.973.5", - "@aws-sdk/util-endpoints": "^3.996.4", - "@smithy/core": "^3.23.8", - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", - "@smithy/util-retry": "^4.2.11", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/types": "^3.973.6", + "@aws-sdk/util-endpoints": "^3.996.5", + "@smithy/core": "^3.23.11", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", + "@smithy/util-retry": "^4.2.12", "tslib": "^2.6.2" }, "engines": { @@ -790,47 +813,47 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.996.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.996.7.tgz", - "integrity": "sha512-MlGWA8uPaOs5AiTZ5JLM4uuWDm9EEAnm9cqwvqQIc6kEgel/8s1BaOWm9QgUcfc9K8qd7KkC3n43yDbeXOA2tg==", + "version": "3.996.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.996.10.tgz", + "integrity": "sha512-SlDol5Z+C7Ivnc2rKGqiqfSUmUZzY1qHfVs9myt/nxVwswgfpjdKahyTzLTx802Zfq0NFRs7AejwKzzzl5Co2w==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/middleware-host-header": "^3.972.7", - "@aws-sdk/middleware-logger": "^3.972.7", - "@aws-sdk/middleware-recursion-detection": "^3.972.7", - "@aws-sdk/middleware-user-agent": "^3.972.19", - "@aws-sdk/region-config-resolver": "^3.972.7", - "@aws-sdk/types": "^3.973.5", - "@aws-sdk/util-endpoints": "^3.996.4", - "@aws-sdk/util-user-agent-browser": "^3.972.7", - "@aws-sdk/util-user-agent-node": "^3.973.4", - "@smithy/config-resolver": "^4.4.10", - "@smithy/core": "^3.23.8", - "@smithy/fetch-http-handler": "^5.3.13", - "@smithy/hash-node": "^4.2.11", - "@smithy/invalid-dependency": "^4.2.11", - "@smithy/middleware-content-length": "^4.2.11", - "@smithy/middleware-endpoint": "^4.4.22", - "@smithy/middleware-retry": "^4.4.39", - "@smithy/middleware-serde": "^4.2.12", - "@smithy/middleware-stack": "^4.2.11", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/node-http-handler": "^4.4.14", - "@smithy/protocol-http": "^5.3.11", - "@smithy/smithy-client": "^4.12.2", - "@smithy/types": "^4.13.0", - "@smithy/url-parser": "^4.2.11", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/middleware-host-header": "^3.972.8", + "@aws-sdk/middleware-logger": "^3.972.8", + "@aws-sdk/middleware-recursion-detection": "^3.972.8", + "@aws-sdk/middleware-user-agent": "^3.972.21", + "@aws-sdk/region-config-resolver": "^3.972.8", + "@aws-sdk/types": "^3.973.6", + "@aws-sdk/util-endpoints": "^3.996.5", + "@aws-sdk/util-user-agent-browser": "^3.972.8", + "@aws-sdk/util-user-agent-node": "^3.973.7", + "@smithy/config-resolver": "^4.4.11", + "@smithy/core": "^3.23.11", + "@smithy/fetch-http-handler": "^5.3.15", + "@smithy/hash-node": "^4.2.12", + "@smithy/invalid-dependency": "^4.2.12", + "@smithy/middleware-content-length": "^4.2.12", + "@smithy/middleware-endpoint": "^4.4.25", + "@smithy/middleware-retry": "^4.4.42", + "@smithy/middleware-serde": "^4.2.14", + "@smithy/middleware-stack": "^4.2.12", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/node-http-handler": "^4.4.16", + "@smithy/protocol-http": "^5.3.12", + "@smithy/smithy-client": "^4.12.5", + "@smithy/types": "^4.13.1", + "@smithy/url-parser": "^4.2.12", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", - "@smithy/util-defaults-mode-browser": "^4.3.38", - "@smithy/util-defaults-mode-node": "^4.2.41", - "@smithy/util-endpoints": "^3.3.2", - "@smithy/util-middleware": "^4.2.11", - "@smithy/util-retry": "^4.2.11", + "@smithy/util-defaults-mode-browser": "^4.3.41", + "@smithy/util-defaults-mode-node": "^4.2.44", + "@smithy/util-endpoints": "^3.3.3", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-retry": "^4.2.12", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -839,15 +862,15 @@ } }, "node_modules/@aws-sdk/region-config-resolver": { - "version": "3.972.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.7.tgz", - "integrity": "sha512-/Ev/6AI8bvt4HAAptzSjThGUMjcWaX3GX8oERkB0F0F9x2dLSBdgFDiyrRz3i0u0ZFZFQ1b28is4QhyqXTUsVA==", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.8.tgz", + "integrity": "sha512-1eD4uhTDeambO/PNIDVG19A6+v4NdD7xzwLHDutHsUqz0B+i661MwQB2eYO4/crcCvCiQG4SRm1k81k54FEIvw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", - "@smithy/config-resolver": "^4.4.10", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/types": "^4.13.0", + "@aws-sdk/types": "^3.973.6", + "@smithy/config-resolver": "^4.4.11", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -855,16 +878,16 @@ } }, "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.6.tgz", - "integrity": "sha512-NnsOQsVmJXy4+IdPFUjRCWPn9qNH1TzS/f7MiWgXeoHs903tJpAWQWQtoFvLccyPoBgomKP9L89RRr2YsT/L0g==", + "version": "3.996.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.8.tgz", + "integrity": "sha512-n1qYFD+tbqZuyskVaxUE+t10AUz9g3qzDw3Tp6QZDKmqsjfDmZBd4GIk2EKJJNtcCBtE5YiUjDYA+3djFAFBBg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-sdk-s3": "^3.972.18", - "@aws-sdk/types": "^3.973.5", - "@smithy/protocol-http": "^5.3.11", - "@smithy/signature-v4": "^5.3.11", - "@smithy/types": "^4.13.0", + "@aws-sdk/middleware-sdk-s3": "^3.972.20", + "@aws-sdk/types": "^3.973.6", + "@smithy/protocol-http": "^5.3.12", + "@smithy/signature-v4": "^5.3.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -872,17 +895,17 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.1004.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1004.0.tgz", - "integrity": "sha512-j9BwZZId9sFp+4GPhf6KrwO8Tben2sXibZA8D1vv2I1zBdvkUHcBA2g4pkqIpTRalMTLC0NPkBPX0gERxfy/iA==", + "version": "3.1009.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1009.0.tgz", + "integrity": "sha512-KCPLuTqN9u0Rr38Arln78fRG9KXpzsPWmof+PZzfAHMMQq2QED6YjQrkrfiH7PDefLWEposY1o4/eGwrmKA4JA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.18", - "@aws-sdk/nested-clients": "^3.996.7", - "@aws-sdk/types": "^3.973.5", - "@smithy/property-provider": "^4.2.11", - "@smithy/shared-ini-file-loader": "^4.4.6", - "@smithy/types": "^4.13.0", + "@aws-sdk/core": "^3.973.20", + "@aws-sdk/nested-clients": "^3.996.10", + "@aws-sdk/types": "^3.973.6", + "@smithy/property-provider": "^4.2.12", + "@smithy/shared-ini-file-loader": "^4.4.7", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -890,12 +913,12 @@ } }, "node_modules/@aws-sdk/types": { - "version": "3.973.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.5.tgz", - "integrity": "sha512-hl7BGwDCWsjH8NkZfx+HgS7H2LyM2lTMAI7ba9c8O0KqdBLTdNJivsHpqjg9rNlAlPyREb6DeDRXUl0s8uFdmQ==", + "version": "3.973.6", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.6.tgz", + "integrity": "sha512-Atfcy4E++beKtwJHiDln2Nby8W/mam64opFPTiHEqgsthqeydFS1pY+OUlN1ouNOmf8ArPU/6cDS65anOP3KQw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -915,15 +938,15 @@ } }, "node_modules/@aws-sdk/util-endpoints": { - "version": "3.996.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.996.4.tgz", - "integrity": "sha512-Hek90FBmd4joCFj+Vc98KLJh73Zqj3s2W56gjAcTkrNLMDI5nIFkG9YpfcJiVI1YlE2Ne1uOQNe+IgQ/Vz2XRA==", + "version": "3.996.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.996.5.tgz", + "integrity": "sha512-Uh93L5sXFNbyR5sEPMzUU8tJ++Ku97EY4udmC01nB8Zu+xfBPwpIwJ6F7snqQeq8h2pf+8SGN5/NoytfKgYPIw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", - "@smithy/types": "^4.13.0", - "@smithy/url-parser": "^4.2.11", - "@smithy/util-endpoints": "^3.3.2", + "@aws-sdk/types": "^3.973.6", + "@smithy/types": "^4.13.1", + "@smithy/url-parser": "^4.2.12", + "@smithy/util-endpoints": "^3.3.3", "tslib": "^2.6.2" }, "engines": { @@ -943,27 +966,28 @@ } }, "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.972.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.7.tgz", - "integrity": "sha512-7SJVuvhKhMF/BkNS1n0QAJYgvEwYbK2QLKBrzDiwQGiTRU6Yf1f3nehTzm/l21xdAOtWSfp2uWSddPnP2ZtsVw==", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.8.tgz", + "integrity": "sha512-B3KGXJviV2u6Cdw2SDY2aDhoJkVfY/Q/Trwk2CMSkikE1Oi6gRzxhvhIfiRpHfmIsAhV4EA54TVEX8K6CbHbkA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.5", - "@smithy/types": "^4.13.0", + "@aws-sdk/types": "^3.973.6", + "@smithy/types": "^4.13.1", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.973.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.4.tgz", - "integrity": "sha512-uqKeLqZ9D3nQjH7HGIERNXK9qnSpUK08l4MlJ5/NZqSSdeJsVANYp437EM9sEzwU28c2xfj2V6qlkqzsgtKs6Q==", + "version": "3.973.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.7.tgz", + "integrity": "sha512-Hz6EZMUAEzqUd7e+vZ9LE7mn+5gMbxltXy18v+YSFY+9LBJz15wkNZvw5JqfX3z0FS9n3bgUtz3L5rAsfh4YlA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-user-agent": "^3.972.19", - "@aws-sdk/types": "^3.973.5", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/types": "^4.13.0", + "@aws-sdk/middleware-user-agent": "^3.972.21", + "@aws-sdk/types": "^3.973.6", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/types": "^4.13.1", + "@smithy/util-config-provider": "^4.2.2", "tslib": "^2.6.2" }, "engines": { @@ -979,12 +1003,12 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.10", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.10.tgz", - "integrity": "sha512-OnejAIVD+CxzyAUrVic7lG+3QRltyja9LoNqCE/1YVs8ichoTbJlVSaZ9iSMcnHLyzrSNtvaOGjSDRP+d/ouFA==", + "version": "3.972.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.11.tgz", + "integrity": "sha512-iitV/gZKQMvY9d7ovmyFnFuTHbBAtrmLnvaSb/3X8vOKyevwtpmEtyc8AdhVWZe0pI/1GsHxlEvQeOePFzy7KQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "fast-xml-parser": "5.4.1", "tslib": "^2.6.2" }, @@ -993,9 +1017,9 @@ } }, "node_modules/@aws/lambda-invoke-store": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.3.tgz", - "integrity": "sha512-oLvsaPMTBejkkmHhjf09xTgk71mOqyr/409NKhRIL08If7AhVfUsJhVsx386uJaqNd42v9kWamQ9lFbkoC2dYw==", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.4.tgz", + "integrity": "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==", "license": "Apache-2.0", "engines": { "node": ">=18.0.0" @@ -6157,12 +6181,12 @@ "license": "(Unlicense OR Apache-2.0)" }, "node_modules/@smithy/abort-controller": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.11.tgz", - "integrity": "sha512-Hj4WoYWMJnSpM6/kchsm4bUNTL9XiSyhvoMb2KIq4VJzyDt7JpGHUZHkVNPZVC7YE1tf8tPeVauxpFBKGW4/KQ==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.12.tgz", + "integrity": "sha512-xolrFw6b+2iYGl6EcOL7IJY71vvyZ0DJ3mcKtpykqPe2uscwtzDZJa1uVQXyP7w9Dd+kGwYnPbMsJrGISKiY/Q==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6195,16 +6219,16 @@ } }, "node_modules/@smithy/config-resolver": { - "version": "4.4.10", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.10.tgz", - "integrity": "sha512-IRTkd6ps0ru+lTWnfnsbXzW80A8Od8p3pYiZnW98K2Hb20rqfsX7VTlfUwhrcOeSSy68Gn9WBofwPuw3e5CCsg==", + "version": "4.4.11", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.11.tgz", + "integrity": "sha512-YxFiiG4YDAtX7WMN7RuhHZLeTmRRAOyCbr+zB8e3AQzHPnUhS8zXjB1+cniPVQI3xbWsQPM0X2aaIkO/ME0ymw==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^4.3.11", - "@smithy/types": "^4.13.0", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/types": "^4.13.1", "@smithy/util-config-provider": "^4.2.2", - "@smithy/util-endpoints": "^3.3.2", - "@smithy/util-middleware": "^4.2.11", + "@smithy/util-endpoints": "^3.3.3", + "@smithy/util-middleware": "^4.2.12", "tslib": "^2.6.2" }, "engines": { @@ -6212,18 +6236,18 @@ } }, "node_modules/@smithy/core": { - "version": "3.23.9", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.23.9.tgz", - "integrity": "sha512-1Vcut4LEL9HZsdpI0vFiRYIsaoPwZLjAxnVQDUMQK8beMS+EYPLDQCXtbzfxmM5GzSgjfe2Q9M7WaXwIMQllyQ==", + "version": "3.23.11", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.23.11.tgz", + "integrity": "sha512-952rGf7hBRnhUIaeLp6q4MptKW8sPFe5VvkoZ5qIzFAtx6c/QZ/54FS3yootsyUSf9gJX/NBqEBNdNR7jMIlpQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/middleware-serde": "^4.2.12", - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", + "@smithy/url-parser": "^4.2.12", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", - "@smithy/util-middleware": "^4.2.11", - "@smithy/util-stream": "^4.5.17", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-stream": "^4.5.19", "@smithy/util-utf8": "^4.2.2", "@smithy/uuid": "^1.1.2", "tslib": "^2.6.2" @@ -6233,15 +6257,15 @@ } }, "node_modules/@smithy/credential-provider-imds": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.11.tgz", - "integrity": "sha512-lBXrS6ku0kTj3xLmsJW0WwqWbGQ6ueooYyp/1L9lkyT0M02C+DWwYwc5aTyXFbRaK38ojALxNixg+LxKSHZc0g==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.12.tgz", + "integrity": "sha512-cr2lR792vNZcYMriSIj+Um3x9KWrjcu98kn234xA6reOAFMmbRpQMOv8KPgEmLLtx3eldU6c5wALKFqNOhugmg==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^4.3.11", - "@smithy/property-provider": "^4.2.11", - "@smithy/types": "^4.13.0", - "@smithy/url-parser": "^4.2.11", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/property-provider": "^4.2.12", + "@smithy/types": "^4.13.1", + "@smithy/url-parser": "^4.2.12", "tslib": "^2.6.2" }, "engines": { @@ -6249,13 +6273,13 @@ } }, "node_modules/@smithy/eventstream-codec": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-4.2.11.tgz", - "integrity": "sha512-Sf39Ml0iVX+ba/bgMPxaXWAAFmHqYLTmbjAPfLPLY8CrYkRDEqZdUsKC1OwVMCdJXfAt0v4j49GIJ8DoSYAe6w==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-4.2.12.tgz", + "integrity": "sha512-FE3bZdEl62ojmy8x4FHqxq2+BuOHlcxiH5vaZ6aqHJr3AIZzwF5jfx8dEiU/X0a8RboyNDjmXjlbr8AdEyLgiA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/crc32": "5.2.0", - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "@smithy/util-hex-encoding": "^4.2.2", "tslib": "^2.6.2" }, @@ -6264,13 +6288,13 @@ } }, "node_modules/@smithy/eventstream-serde-browser": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.2.11.tgz", - "integrity": "sha512-3rEpo3G6f/nRS7fQDsZmxw/ius6rnlIpz4UX6FlALEzz8JoSxFmdBt0SZnthis+km7sQo6q5/3e+UJcuQivoXA==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.2.12.tgz", + "integrity": "sha512-XUSuMxlTxV5pp4VpqZf6Sa3vT/Q75FVkLSpSSE3KkWBvAQWeuWt1msTv8fJfgA4/jcJhrbrbMzN1AC/hvPmm5A==", "license": "Apache-2.0", "dependencies": { - "@smithy/eventstream-serde-universal": "^4.2.11", - "@smithy/types": "^4.13.0", + "@smithy/eventstream-serde-universal": "^4.2.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6278,12 +6302,12 @@ } }, "node_modules/@smithy/eventstream-serde-config-resolver": { - "version": "4.3.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.3.11.tgz", - "integrity": "sha512-XeNIA8tcP/GDWnnKkO7qEm/bg0B/bP9lvIXZBXcGZwZ+VYM8h8k9wuDvUODtdQ2Wcp2RcBkPTCSMmaniVHrMlA==", + "version": "4.3.12", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.3.12.tgz", + "integrity": "sha512-7epsAZ3QvfHkngz6RXQYseyZYHlmWXSTPOfPmXkiS+zA6TBNo1awUaMFL9vxyXlGdoELmCZyZe1nQE+imbmV+Q==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6291,13 +6315,13 @@ } }, "node_modules/@smithy/eventstream-serde-node": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.2.11.tgz", - "integrity": "sha512-fzbCh18rscBDTQSCrsp1fGcclLNF//nJyhjldsEl/5wCYmgpHblv5JSppQAyQI24lClsFT0wV06N1Porn0IsEw==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.2.12.tgz", + "integrity": "sha512-D1pFuExo31854eAvg89KMn9Oab/wEeJR6Buy32B49A9Ogdtx5fwZPqBHUlDzaCDpycTFk2+fSQgX689Qsk7UGA==", "license": "Apache-2.0", "dependencies": { - "@smithy/eventstream-serde-universal": "^4.2.11", - "@smithy/types": "^4.13.0", + "@smithy/eventstream-serde-universal": "^4.2.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6305,13 +6329,13 @@ } }, "node_modules/@smithy/eventstream-serde-universal": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.2.11.tgz", - "integrity": "sha512-MJ7HcI+jEkqoWT5vp+uoVaAjBrmxBtKhZTeynDRG/seEjJfqyg3SiqMMqyPnAMzmIfLaeJ/uiuSDP/l9AnMy/Q==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.2.12.tgz", + "integrity": "sha512-+yNuTiyBACxOJUTvbsNsSOfH9G9oKbaJE1lNL3YHpGcuucl6rPZMi3nrpehpVOVR2E07YqFFmtwpImtpzlouHQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/eventstream-codec": "^4.2.11", - "@smithy/types": "^4.13.0", + "@smithy/eventstream-codec": "^4.2.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6319,14 +6343,14 @@ } }, "node_modules/@smithy/fetch-http-handler": { - "version": "5.3.13", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.13.tgz", - "integrity": "sha512-U2Hcfl2s3XaYjikN9cT4mPu8ybDbImV3baXR0PkVlC0TTx808bRP3FaPGAzPtB8OByI+JqJ1kyS+7GEgae7+qQ==", + "version": "5.3.15", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.15.tgz", + "integrity": "sha512-T4jFU5N/yiIfrtrsb9uOQn7RdELdM/7HbyLNr6uO/mpkj1ctiVs7CihVr51w4LyQlXWDpXFn4BElf1WmQvZu/A==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^5.3.11", - "@smithy/querystring-builder": "^4.2.11", - "@smithy/types": "^4.13.0", + "@smithy/protocol-http": "^5.3.12", + "@smithy/querystring-builder": "^4.2.12", + "@smithy/types": "^4.13.1", "@smithy/util-base64": "^4.3.2", "tslib": "^2.6.2" }, @@ -6335,14 +6359,14 @@ } }, "node_modules/@smithy/hash-blob-browser": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-4.2.12.tgz", - "integrity": "sha512-1wQE33DsxkM/waftAhCH9VtJbUGyt1PJ9YRDpOu+q9FUi73LLFUZ2fD8A61g2mT1UY9k7b99+V1xZ41Rz4SHRQ==", + "version": "4.2.13", + "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-4.2.13.tgz", + "integrity": "sha512-YrF4zWKh+ghLuquldj6e/RzE3xZYL8wIPfkt0MqCRphVICjyyjH8OwKD7LLlKpVEbk4FLizFfC1+gwK6XQdR3g==", "license": "Apache-2.0", "dependencies": { "@smithy/chunked-blob-reader": "^5.2.2", "@smithy/chunked-blob-reader-native": "^4.2.3", - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6350,12 +6374,12 @@ } }, "node_modules/@smithy/hash-node": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.11.tgz", - "integrity": "sha512-T+p1pNynRkydpdL015ruIoyPSRw9e/SQOWmSAMmmprfswMrd5Ow5igOWNVlvyVFZlxXqGmyH3NQwfwy8r5Jx0A==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.12.tgz", + "integrity": "sha512-QhBYbGrbxTkZ43QoTPrK72DoYviDeg6YKDrHTMJbbC+A0sml3kSjzFtXP7BtbyJnXojLfTQldGdUR0RGD8dA3w==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "@smithy/util-buffer-from": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" @@ -6365,12 +6389,12 @@ } }, "node_modules/@smithy/hash-stream-node": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-4.2.11.tgz", - "integrity": "sha512-hQsTjwPCRY8w9GK07w1RqJi3e+myh0UaOWBBhZ1UMSDgofH/Q1fEYzU1teaX6HkpX/eWDdm7tAGR0jBPlz9QEQ==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-4.2.12.tgz", + "integrity": "sha512-O3YbmGExeafuM/kP7Y8r6+1y0hIh3/zn6GROx0uNlB54K9oihAL75Qtc+jFfLNliTi6pxOAYZrRKD9A7iA6UFw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -6379,12 +6403,12 @@ } }, "node_modules/@smithy/invalid-dependency": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.11.tgz", - "integrity": "sha512-cGNMrgykRmddrNhYy1yBdrp5GwIgEkniS7k9O1VLB38yxQtlvrxpZtUVvo6T4cKpeZsriukBuuxfJcdZQc/f/g==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.12.tgz", + "integrity": "sha512-/4F1zb7Z8LOu1PalTdESFHR0RbPwHd3FcaG1sI3UEIriQTWakysgJr65lc1jj6QY5ye7aFsisajotH6UhWfm/g==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6404,12 +6428,12 @@ } }, "node_modules/@smithy/md5-js": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-4.2.11.tgz", - "integrity": "sha512-350X4kGIrty0Snx2OWv7rPM6p6vM7RzryvFs6B/56Cux3w3sChOb3bymo5oidXJlPcP9fIRxGUCk7GqpiSOtng==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-4.2.12.tgz", + "integrity": "sha512-W/oIpHCpWU2+iAkfZYyGWE+qkpuf3vEXHLxQQDx9FPNZTTdnul0dZ2d/gUFrtQ5je1G2kp4cjG0/24YueG2LbQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, @@ -6418,13 +6442,13 @@ } }, "node_modules/@smithy/middleware-content-length": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.11.tgz", - "integrity": "sha512-UvIfKYAKhCzr4p6jFevPlKhQwyQwlJ6IeKLDhmV1PlYfcW3RL4ROjNEDtSik4NYMi9kDkH7eSwyTP3vNJ/u/Dw==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.12.tgz", + "integrity": "sha512-YE58Yz+cvFInWI/wOTrB+DbvUVz/pLn5mC5MvOV4fdRUc6qGwygyngcucRQjAhiCEbmfLOXX0gntSIcgMvAjmA==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6432,18 +6456,18 @@ } }, "node_modules/@smithy/middleware-endpoint": { - "version": "4.4.23", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.23.tgz", - "integrity": "sha512-UEFIejZy54T1EJn2aWJ45voB7RP2T+IRzUqocIdM6GFFa5ClZncakYJfcYnoXt3UsQrZZ9ZRauGm77l9UCbBLw==", + "version": "4.4.25", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.25.tgz", + "integrity": "sha512-dqjLwZs2eBxIUG6Qtw8/YZ4DvzHGIf0DA18wrgtfP6a50UIO7e2nY0FPdcbv5tVJKqWCCU5BmGMOUwT7Puan+A==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.9", - "@smithy/middleware-serde": "^4.2.12", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/shared-ini-file-loader": "^4.4.6", - "@smithy/types": "^4.13.0", - "@smithy/url-parser": "^4.2.11", - "@smithy/util-middleware": "^4.2.11", + "@smithy/core": "^3.23.11", + "@smithy/middleware-serde": "^4.2.14", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/shared-ini-file-loader": "^4.4.7", + "@smithy/types": "^4.13.1", + "@smithy/url-parser": "^4.2.12", + "@smithy/util-middleware": "^4.2.12", "tslib": "^2.6.2" }, "engines": { @@ -6451,18 +6475,18 @@ } }, "node_modules/@smithy/middleware-retry": { - "version": "4.4.40", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.40.tgz", - "integrity": "sha512-YhEMakG1Ae57FajERdHNZ4ShOPIY7DsgV+ZoAxo/5BT0KIe+f6DDU2rtIymNNFIj22NJfeeI6LWIifrwM0f+rA==", + "version": "4.4.42", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.42.tgz", + "integrity": "sha512-vbwyqHRIpIZutNXZpLAozakzamcINaRCpEy1MYmK6xBeW3xN+TyPRA123GjXnuxZIjc9848MRRCugVMTXxC4Eg==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^4.3.11", - "@smithy/protocol-http": "^5.3.11", - "@smithy/service-error-classification": "^4.2.11", - "@smithy/smithy-client": "^4.12.3", - "@smithy/types": "^4.13.0", - "@smithy/util-middleware": "^4.2.11", - "@smithy/util-retry": "^4.2.11", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/service-error-classification": "^4.2.12", + "@smithy/smithy-client": "^4.12.5", + "@smithy/types": "^4.13.1", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-retry": "^4.2.12", "@smithy/uuid": "^1.1.2", "tslib": "^2.6.2" }, @@ -6471,13 +6495,14 @@ } }, "node_modules/@smithy/middleware-serde": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.12.tgz", - "integrity": "sha512-W9g1bOLui7Xn5FABRVS0o3rXL0gfN37d/8I/W7i0N7oxjx9QecUmXEMSUMADTODwdtka9cN43t5BI2CodLJpng==", + "version": "4.2.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.14.tgz", + "integrity": "sha512-+CcaLoLa5apzSRtloOyG7lQvkUw2ZDml3hRh4QiG9WyEPfW5Ke/3tPOPiPjUneuT59Tpn8+c3RVaUvvkkwqZwg==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", + "@smithy/core": "^3.23.11", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6485,12 +6510,12 @@ } }, "node_modules/@smithy/middleware-stack": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.11.tgz", - "integrity": "sha512-s+eenEPW6RgliDk2IhjD2hWOxIx1NKrOHxEwNUaUXxYBxIyCcDfNULZ2Mu15E3kwcJWBedTET/kEASPV1A1Akg==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.12.tgz", + "integrity": "sha512-kruC5gRHwsCOuyCd4ouQxYjgRAym2uDlCvQ5acuMtRrcdfg7mFBg6blaxcJ09STpt3ziEkis6bhg1uwrWU7txw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6498,14 +6523,14 @@ } }, "node_modules/@smithy/node-config-provider": { - "version": "4.3.11", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.11.tgz", - "integrity": "sha512-xD17eE7kaLgBBGf5CZQ58hh2YmwK1Z0O8YhffwB/De2jsL0U3JklmhVYJ9Uf37OtUDLF2gsW40Xwwag9U869Gg==", + "version": "4.3.12", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.12.tgz", + "integrity": "sha512-tr2oKX2xMcO+rBOjobSwVAkV05SIfUKz8iI53rzxEmgW3GOOPOv0UioSDk+J8OpRQnpnhsO3Af6IEBabQBVmiw==", "license": "Apache-2.0", "dependencies": { - "@smithy/property-provider": "^4.2.11", - "@smithy/shared-ini-file-loader": "^4.4.6", - "@smithy/types": "^4.13.0", + "@smithy/property-provider": "^4.2.12", + "@smithy/shared-ini-file-loader": "^4.4.7", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6513,15 +6538,15 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "4.4.14", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.14.tgz", - "integrity": "sha512-DamSqaU8nuk0xTJDrYnRzZndHwwRnyj/n/+RqGGCcBKB4qrQem0mSDiWdupaNWdwxzyMU91qxDmHOCazfhtO3A==", + "version": "4.4.16", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.16.tgz", + "integrity": "sha512-ULC8UCS/HivdCB3jhi+kLFYe4B5gxH2gi9vHBfEIiRrT2jfKiZNiETJSlzRtE6B26XbBHjPtc8iZKSNqMol9bw==", "license": "Apache-2.0", "dependencies": { - "@smithy/abort-controller": "^4.2.11", - "@smithy/protocol-http": "^5.3.11", - "@smithy/querystring-builder": "^4.2.11", - "@smithy/types": "^4.13.0", + "@smithy/abort-controller": "^4.2.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/querystring-builder": "^4.2.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6529,12 +6554,12 @@ } }, "node_modules/@smithy/property-provider": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.11.tgz", - "integrity": "sha512-14T1V64o6/ndyrnl1ze1ZhyLzIeYNN47oF/QU6P5m82AEtyOkMJTb0gO1dPubYjyyKuPD6OSVMPDKe+zioOnCg==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.12.tgz", + "integrity": "sha512-jqve46eYU1v7pZ5BM+fmkbq3DerkSluPr5EhvOcHxygxzD05ByDRppRwRPPpFrsFo5yDtCYLKu+kreHKVrvc7A==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6542,12 +6567,12 @@ } }, "node_modules/@smithy/protocol-http": { - "version": "5.3.11", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.11.tgz", - "integrity": "sha512-hI+barOVDJBkNt4y0L2mu3Ugc0w7+BpJ2CZuLwXtSltGAAwCb3IvnalGlbDV/UCS6a9ZuT3+exd1WxNdLb5IlQ==", + "version": "5.3.12", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.12.tgz", + "integrity": "sha512-fit0GZK9I1xoRlR4jXmbLhoN0OdEpa96ul8M65XdmXnxXkuMxM0Y8HDT0Fh0Xb4I85MBvBClOzgSrV1X2s1Hxw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6555,12 +6580,12 @@ } }, "node_modules/@smithy/querystring-builder": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.11.tgz", - "integrity": "sha512-7spdikrYiljpket6u0up2Ck2mxhy7dZ0+TDd+S53Dg2DHd6wg+YNJrTCHiLdgZmEXZKI7LJZcwL3721ZRDFiqA==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.12.tgz", + "integrity": "sha512-6wTZjGABQufekycfDGMEB84BgtdOE/rCVTov+EDXQ8NHKTUNIp/j27IliwP7tjIU9LR+sSzyGBOXjeEtVgzCHg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "@smithy/util-uri-escape": "^4.2.2", "tslib": "^2.6.2" }, @@ -6569,12 +6594,12 @@ } }, "node_modules/@smithy/querystring-parser": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.11.tgz", - "integrity": "sha512-nE3IRNjDltvGcoThD2abTozI1dkSy8aX+a2N1Rs55en5UsdyyIXgGEmevUL3okZFoJC77JgRGe99xYohhsjivQ==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.12.tgz", + "integrity": "sha512-P2OdvrgiAKpkPNKlKUtWbNZKB1XjPxM086NeVhK+W+wI46pIKdWBe5QyXvhUm3MEcyS/rkLvY8rZzyUdmyDZBw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6582,24 +6607,24 @@ } }, "node_modules/@smithy/service-error-classification": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.11.tgz", - "integrity": "sha512-HkMFJZJUhzU3HvND1+Yw/kYWXp4RPDLBWLcK1n+Vqw8xn4y2YiBhdww8IxhkQjP/QlZun5bwm3vcHc8AqIU3zw==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.12.tgz", + "integrity": "sha512-LlP29oSQN0Tw0b6D0Xo6BIikBswuIiGYbRACy5ujw/JgWSzTdYj46U83ssf6Ux0GyNJVivs2uReU8pt7Eu9okQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0" + "@smithy/types": "^4.13.1" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@smithy/shared-ini-file-loader": { - "version": "4.4.6", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.6.tgz", - "integrity": "sha512-IB/M5I8G0EeXZTHsAxpx51tMQ5R719F3aq+fjEB6VtNcCHDc0ajFDIGDZw+FW9GxtEkgTduiPpjveJdA/CX7sw==", + "version": "4.4.7", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.7.tgz", + "integrity": "sha512-HrOKWsUb+otTeo1HxVWeEb99t5ER1XrBi/xka2Wv6NVmTbuCUC1dvlrksdvxFtODLBjsC+PHK+fuy2x/7Ynyiw==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6607,16 +6632,16 @@ } }, "node_modules/@smithy/signature-v4": { - "version": "5.3.11", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.11.tgz", - "integrity": "sha512-V1L6N9aKOBAN4wEHLyqjLBnAz13mtILU0SeDrjOaIZEeN6IFa6DxwRt1NNpOdmSpQUfkBj0qeD3m6P77uzMhgQ==", + "version": "5.3.12", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.12.tgz", + "integrity": "sha512-B/FBwO3MVOL00DaRSXfXfa/TRXRheagt/q5A2NM13u7q+sHS59EOVGQNfG7DkmVtdQm5m3vOosoKAXSqn/OEgw==", "license": "Apache-2.0", "dependencies": { "@smithy/is-array-buffer": "^4.2.2", - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", "@smithy/util-hex-encoding": "^4.2.2", - "@smithy/util-middleware": "^4.2.11", + "@smithy/util-middleware": "^4.2.12", "@smithy/util-uri-escape": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" @@ -6626,17 +6651,17 @@ } }, "node_modules/@smithy/smithy-client": { - "version": "4.12.3", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.12.3.tgz", - "integrity": "sha512-7k4UxjSpHmPN2AxVhvIazRSzFQjWnud3sOsXcFStzagww17j1cFQYqTSiQ8xuYK3vKLR1Ni8FzuT3VlKr3xCNw==", + "version": "4.12.5", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.12.5.tgz", + "integrity": "sha512-UqwYawyqSr/aog8mnLnfbPurS0gi4G7IYDcD28cUIBhsvWs1+rQcL2IwkUQ+QZ7dibaoRzhNF99fAQ9AUcO00w==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.9", - "@smithy/middleware-endpoint": "^4.4.23", - "@smithy/middleware-stack": "^4.2.11", - "@smithy/protocol-http": "^5.3.11", - "@smithy/types": "^4.13.0", - "@smithy/util-stream": "^4.5.17", + "@smithy/core": "^3.23.11", + "@smithy/middleware-endpoint": "^4.4.25", + "@smithy/middleware-stack": "^4.2.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", + "@smithy/util-stream": "^4.5.19", "tslib": "^2.6.2" }, "engines": { @@ -6644,9 +6669,9 @@ } }, "node_modules/@smithy/types": { - "version": "4.13.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.13.0.tgz", - "integrity": "sha512-COuLsZILbbQsdrwKQpkkpyep7lCsByxwj7m0Mg5v66/ZTyenlfBc40/QFQ5chO0YN/PNEH1Bi3fGtfXPnYNeDw==", + "version": "4.13.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.13.1.tgz", + "integrity": "sha512-787F3yzE2UiJIQ+wYW1CVg2odHjmaWLGksnKQHUrK/lYZSEcy1msuLVvxaR/sI2/aDe9U+TBuLsXnr3vod1g0g==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -6656,13 +6681,13 @@ } }, "node_modules/@smithy/url-parser": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.11.tgz", - "integrity": "sha512-oTAGGHo8ZYc5VZsBREzuf5lf2pAurJQsccMusVZ85wDkX66ojEc/XauiGjzCj50A61ObFTPe6d7Pyt6UBYaing==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.12.tgz", + "integrity": "sha512-wOPKPEpso+doCZGIlr+e1lVI6+9VAKfL4kZWFgzVgGWY2hZxshNKod4l2LXS3PRC9otH/JRSjtEHqQ/7eLciRA==", "license": "Apache-2.0", "dependencies": { - "@smithy/querystring-parser": "^4.2.11", - "@smithy/types": "^4.13.0", + "@smithy/querystring-parser": "^4.2.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6733,14 +6758,14 @@ } }, "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.39", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.39.tgz", - "integrity": "sha512-ui7/Ho/+VHqS7Km2wBw4/Ab4RktoiSshgcgpJzC4keFPs6tLJS4IQwbeahxQS3E/w98uq6E1mirCH/id9xIXeQ==", + "version": "4.3.41", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.41.tgz", + "integrity": "sha512-M1w1Ux0rSVvBOxIIiqbxvZvhnjQ+VUjJrugtORE90BbadSTH+jsQL279KRL3Hv0w69rE7EuYkV/4Lepz/NBW9g==", "license": "Apache-2.0", "dependencies": { - "@smithy/property-provider": "^4.2.11", - "@smithy/smithy-client": "^4.12.3", - "@smithy/types": "^4.13.0", + "@smithy/property-provider": "^4.2.12", + "@smithy/smithy-client": "^4.12.5", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6748,17 +6773,17 @@ } }, "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.42", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.42.tgz", - "integrity": "sha512-QDA84CWNe8Akpj15ofLO+1N3Rfg8qa2K5uX0y6HnOp4AnRYRgWrKx/xzbYNbVF9ZsyJUYOfcoaN3y93wA/QJ2A==", + "version": "4.2.44", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.44.tgz", + "integrity": "sha512-YPze3/lD1KmWuZsl9JlfhcgGLX7AXhSoaCDtiPntUjNW5/YY0lOHjkcgxyE9x/h5vvS1fzDifMGjzqnNlNiqOQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/config-resolver": "^4.4.10", - "@smithy/credential-provider-imds": "^4.2.11", - "@smithy/node-config-provider": "^4.3.11", - "@smithy/property-provider": "^4.2.11", - "@smithy/smithy-client": "^4.12.3", - "@smithy/types": "^4.13.0", + "@smithy/config-resolver": "^4.4.11", + "@smithy/credential-provider-imds": "^4.2.12", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/property-provider": "^4.2.12", + "@smithy/smithy-client": "^4.12.5", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6766,13 +6791,13 @@ } }, "node_modules/@smithy/util-endpoints": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.3.2.tgz", - "integrity": "sha512-+4HFLpE5u29AbFlTdlKIT7jfOzZ8PDYZKTb3e+AgLz986OYwqTourQ5H+jg79/66DB69Un1+qKecLnkZdAsYcA==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.3.3.tgz", + "integrity": "sha512-VACQVe50j0HZPjpwWcjyT51KUQ4AnsvEaQ2lKHOSL4mNLD0G9BjEniQ+yCt1qqfKfiAHRAts26ud7hBjamrwig==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^4.3.11", - "@smithy/types": "^4.13.0", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6792,12 +6817,12 @@ } }, "node_modules/@smithy/util-middleware": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.11.tgz", - "integrity": "sha512-r3dtF9F+TpSZUxpOVVtPfk09Rlo4lT6ORBqEvX3IBT6SkQAdDSVKR5GcfmZbtl7WKhKnmb3wbDTQ6ibR2XHClw==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.12.tgz", + "integrity": "sha512-Er805uFUOvgc0l8nv0e0su0VFISoxhJ/AwOn3gL2NWNY2LUEldP5WtVcRYSQBcjg0y9NfG8JYrCJaYDpupBHJQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.13.0", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6805,13 +6830,13 @@ } }, "node_modules/@smithy/util-retry": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.2.11.tgz", - "integrity": "sha512-XSZULmL5x6aCTTii59wJqKsY1l3eMIAomRAccW7Tzh9r8s7T/7rdo03oektuH5jeYRlJMPcNP92EuRDvk9aXbw==", + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.2.12.tgz", + "integrity": "sha512-1zopLDUEOwumjcHdJ1mwBHddubYF8GMQvstVCLC54Y46rqoHwlIU+8ZzUeaBcD+WCJHyDGSeZ2ml9YSe9aqcoQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/service-error-classification": "^4.2.11", - "@smithy/types": "^4.13.0", + "@smithy/service-error-classification": "^4.2.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -6819,14 +6844,14 @@ } }, "node_modules/@smithy/util-stream": { - "version": "4.5.17", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.17.tgz", - "integrity": "sha512-793BYZ4h2JAQkNHcEnyFxDTcZbm9bVybD0UV/LEWmZ5bkTms7JqjfrLMi2Qy0E5WFcCzLwCAPgcvcvxoeALbAQ==", + "version": "4.5.19", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.19.tgz", + "integrity": "sha512-v4sa+3xTweL1CLO2UP0p7tvIMH/Rq1X4KKOxd568mpe6LSLMQCnDHs4uv7m3ukpl3HvcN2JH6jiCS0SNRXKP/w==", "license": "Apache-2.0", "dependencies": { - "@smithy/fetch-http-handler": "^5.3.13", - "@smithy/node-http-handler": "^4.4.14", - "@smithy/types": "^4.13.0", + "@smithy/fetch-http-handler": "^5.3.15", + "@smithy/node-http-handler": "^4.4.16", + "@smithy/types": "^4.13.1", "@smithy/util-base64": "^4.3.2", "@smithy/util-buffer-from": "^4.2.2", "@smithy/util-hex-encoding": "^4.2.2", @@ -6863,13 +6888,13 @@ } }, "node_modules/@smithy/util-waiter": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-4.2.11.tgz", - "integrity": "sha512-x7Rh2azQPs3XxbvCzcttRErKKvLnbZfqRf/gOjw2pb+ZscX88e5UkRPCB67bVnsFHxayvMvmePfKTqsRb+is1A==", + "version": "4.2.13", + "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-4.2.13.tgz", + "integrity": "sha512-2zdZ9DTHngRtcYxJK1GUDxruNr53kv5W2Lupe0LMU+Imr6ohQg8M2T14MNkj1Y0wS3FFwpgpGQyvuaMF7CiTmQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/abort-controller": "^4.2.11", - "@smithy/types": "^4.13.0", + "@smithy/abort-controller": "^4.2.12", + "@smithy/types": "^4.13.1", "tslib": "^2.6.2" }, "engines": { @@ -8526,10 +8551,9 @@ } }, "node_modules/basic-ftp": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.1.0.tgz", - "integrity": "sha512-RkaJzeJKDbaDWTIPiJwubyljaEPwpVWkm9Rt5h9Nd6h7tEXTJ3VB4qxdZBioV7JO5yLUaOKwz7vDOzlncUsegw==", - "dev": true, + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.0.tgz", + "integrity": "sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -8785,6 +8809,16 @@ "safe-buffer": "^5.1.2" } }, + "node_modules/buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -11896,6 +11930,15 @@ "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", "license": "MIT" }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/evp_bytestokey": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", @@ -12134,16 +12177,19 @@ "license": "MIT" }, "node_modules/fast-xml-builder": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.0.0.tgz", - "integrity": "sha512-fpZuDogrAgnyt9oDDz+5DBz0zgPdPZz6D4IR7iESxRXElrlGTRkHJ9eEt+SACRJwT0FNFrt71DFQIUFBJfX/uQ==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.3.tgz", + "integrity": "sha512-1o60KoFw2+LWKQu3IdcfcFlGTW4dpqEWmjhYec6H82AYZU2TVBXep6tMl8Z1Y+wM+ZrzCwe3BZ9Vyd9N2rIvmg==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT" + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.1.3" + } }, "node_modules/fast-xml-parser": { "version": "5.4.1", @@ -17386,6 +17432,21 @@ "node": ">=8" } }, + "node_modules/path-expression-matcher": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.1.3.tgz", + "integrity": "sha512-qdVgY8KXmVdJZRSS1JdEPOKPdTiEK/pi0RkcT2sw1RhXxohdujUlJFPuS1TSkevZ9vzd3ZlL7ULl1MHGTApKzQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -18305,15 +18366,6 @@ "ieee754": "^1.2.1" } }, - "node_modules/readable-stream/node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "license": "MIT", - "engines": { - "node": ">=0.8.x" - } - }, "node_modules/readable-stream/node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -19564,6 +19616,30 @@ "node": ">= 0.4" } }, + "node_modules/stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "license": "MIT", + "dependencies": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, + "node_modules/stream-browserify/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/stream-chunks": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stream-chunks/-/stream-chunks-1.0.0.tgz", diff --git a/package.json b/package.json index 876f1d41b..7379d23c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ocean-node", - "version": "1.0.7", + "version": "2.0.2", "description": "Ocean Node is used to run all core services in the Ocean stack", "author": "Ocean Protocol Foundation", "license": "Apache-2.0", @@ -49,6 +49,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.1002.0", + "@aws-sdk/lib-storage": "^3.1002.0", "@chainsafe/libp2p-noise": "^17.0.0", "@chainsafe/libp2p-yamux": "^8.0.1", "@elastic/elasticsearch": "^8.14.0", @@ -76,6 +77,7 @@ "@oceanprotocol/ddo-js": "^0.2.0", "axios": "^1.13.5", "base58-js": "^2.0.0", + "basic-ftp": "^5.2.0", "cors": "^2.8.5", "datastore-level": "^12.0.2", "delay": "^5.0.0", @@ -96,7 +98,7 @@ "node-cron": "^3.0.3", "sqlite3": "^5.1.7", "stream-concat": "^1.0.0", - "tar": "^7.5.10", + "tar": "^7.5.11", "uint8arrays": "^4.0.6", "url-join": "^5.0.0", "winston": "^3.11.0", diff --git a/scripts/list_gpus.sh b/scripts/list_gpus.sh index dc3fdf1e1..2abe6781d 100755 --- a/scripts/list_gpus.sh +++ b/scripts/list_gpus.sh @@ -5,29 +5,151 @@ get_nvidia_gpus() { if command -v nvidia-smi &> /dev/null; then # Query nvidia-smi for GPU count, names, and UUIDs # We use csv format for easier parsing - nvidia-smi --query-gpu=name,uuid --format=csv,noheader | while IFS=, read -r name uuid; do + nvidia-smi --query-gpu=name,uuid,driver_version,memory.total --format=csv,noheader | while IFS=, read -r name uuid driver_version memory_total; do # Trim leading/trailing whitespace name=$(echo "$name" | xargs) uuid=$(echo "$uuid" | xargs) + driver_version=$(echo "$driver_version" | xargs) + memory_total=$(echo "$memory_total" | xargs) # Create a JSON object for this GPU # Note: We use the UUID as the ID locally, but it will be aggregated later jq -c -n \ --arg name "$name" \ --arg uuid "$uuid" \ + --arg driver_version "$driver_version" \ + --arg memory_total "$memory_total" \ '{ description: $name, + driverVersion: (if $driver_version != "" then $driver_version else null end), + memoryTotal: (if $memory_total != "" then $memory_total else null end), init: { deviceRequests: { Driver: "nvidia", Devices: [$uuid] } } - }' + } | del(.. | select(. == null))' done fi } +get_driver_version() { + local module="$1" + + # Try the version field first (present when built as a loadable module) + local ver + ver=$(modinfo "$module" 2>/dev/null | awk '/^version:/ {print $2; exit}') + if [ -n "$ver" ]; then + echo "$ver" + return + fi + + # If module exists but only has srcversion (built-in to kernel - integrated + # GPU), fall back to the kernel version as the effective driver version + if modinfo "$module" 2>/dev/null | grep -q "^srcversion:"; then + uname -r + return + fi +} + +get_intel_driver_version() { + # oneAPI / Level Zero runtime (Intel Arc) + local ver + + # Try clinfo first (works for both oneAPI and standard OpenCL) + ver=$(clinfo 2>/dev/null | awk '/Driver Version/ {print $NF; exit}') + [ -n "$ver" ] && { echo "$ver"; return; } + + # Try package manager + ver=$(dpkg -l 2>/dev/null | awk '/intel-level-zero-gpu/ {print $3; exit}') + [ -z "$ver" ] && ver=$(rpm -q --qf '%{VERSION}' intel-level-zero-gpu 2>/dev/null) + [ -n "$ver" ] && { echo "$ver"; return; } + + # Fall back to kernel module / kernel version + get_driver_version "xe" || get_driver_version "i915" +} + +get_amd_driver_version() { + local ver + + # ROCm version file (most reliable) + ver=$(cat /opt/rocm/.info/version 2>/dev/null | head -n1) + [ -n "$ver" ] && { echo "ROCm $ver"; return; } + + # Try rocm-smi + ver=$(rocm-smi --version 2>/dev/null | awk '/ROCm/ {print $NF; exit}') + [ -n "$ver" ] && { echo "$ver"; return; } + + # Try package manager + ver=$(dpkg -l 2>/dev/null | awk '/rocm-core/ {print $3; exit}') + [ -z "$ver" ] && ver=$(rpm -q --qf '%{VERSION}' rocm-core 2>/dev/null) + [ -n "$ver" ] && { echo "ROCm $ver"; return; } + + # Fall back to kernel module + get_driver_version "amdgpu" +} + +get_amd_vram() { + local card_path="$1" + local real_device_path + real_device_path=$(readlink -f "$card_path/device") + + # Try sysfs VRAM total (bytes) — most reliable on amdgpu + local vram_bytes="" + for f in \ + "$real_device_path/mem_info_vram_total" \ + "$card_path/device/mem_info_vram_total"; do + if [ -r "$f" ]; then + vram_bytes=$(cat "$f" 2>/dev/null) + break + fi + done + + if [ -n "$vram_bytes" ] && [ "$vram_bytes" -gt 0 ] 2>/dev/null; then + echo $(( vram_bytes / 1024 / 1024 )) MiB + fi +} + +get_intel_vram() { + local card_path="$1" + local real_device_path + real_device_path=$(readlink -f "$card_path/device") + + # Dedicated VRAM via sysfs (Intel Arc, etc.) + for f in \ + "$real_device_path/drm/$(basename $card_path)/gt/gt0/mem_info_vram_total" \ + "$real_device_path/mem_info_vram_total"; do + if [ -r "$f" ]; then + local bytes + bytes=$(cat "$f" 2>/dev/null) + if [ -n "$bytes" ] && [ "$bytes" -gt 0 ] 2>/dev/null; then + echo "$(( bytes / 1024 / 1024 )) MiB" + return + fi + fi + done + + # Integrated Intel: stolen/GTT memory from lspci, normalize to MiB + local slot + slot=$(basename "$real_device_path") + local raw + raw=$(lspci -s "$slot" -v 2>/dev/null | awk '/Memory at|prefetchable/ && /size=/ { + match($0, /size=([0-9]+[KMG])/, a); if (a[1]) { print a[1]; exit } + }') + + if [ -n "$raw" ]; then + local num unit + num=$(echo "$raw" | tr -d 'KMG') + unit=$(echo "$raw" | tr -d '0-9') + case "$unit" in + K) echo "$(( num / 1024 )) MiB" ;; + M) echo "${num} MiB" ;; + G) echo "$(( num * 1024 )) MiB" ;; + esac + fi +} + # Declare the associative array (hashmap) globally declare -A gpu_map @@ -55,8 +177,6 @@ map_pci_to_primary() { done } - - # Function to check for other GPUs (AMD, Intel, etc.) via lspci get_generic_gpus() { # Check if lspci is available @@ -131,6 +251,11 @@ process_pci_line() { case "$vendor_id_hex" in "1002") # AMD (0x1002) + local amd_driver_version + amd_driver_version=$(get_amd_driver_version) + local amd_memory_total + amd_memory_total=$(get_amd_vram "$card_path") + # Devices if [ -e "/dev/dxg" ]; then devices+=("/dev/dxg") @@ -153,6 +278,11 @@ process_pci_line() { ;; "8086") # Intel (0x8086) + local intel_driver_version + intel_driver_version=$(get_intel_driver_version) + local intel_memory_total + intel_memory_total=$(get_intel_vram "$card_path") + # Devices [ -n "$render_name" ] && devices+=("/dev/dri/$render_name") devices+=("$device_id") @@ -190,11 +320,18 @@ process_pci_line() { json_devices="[\"$device_id\"]" fi - + local driver_version="" + local memory_total="" + case "$vendor_id_hex" in + "1002") driver_version="$amd_driver_version"; memory_total="$amd_memory_total" ;; + "8086") driver_version="$intel_driver_version"; memory_total="$intel_memory_total" ;; + esac jq -c -n \ --arg desc "$description" \ --arg driver "$driver" \ --arg device_id "$device_id" \ + --arg driver_version "$driver_version" \ + --arg memory_total "$memory_total" \ --argjson dev "$json_devices" \ --argjson bind "$json_binds" \ --argjson cap "$json_cap" \ @@ -204,6 +341,8 @@ process_pci_line() { --argjson ipc "$ipc_mode" \ '{ description: $desc, + driverVersion: (if $driver_version != "" then $driver_version else null end), + memoryTotal: (if $memory_total != "" then $memory_total else null end), init: { deviceRequests: { Driver: (if $driver != "" then $driver else null end), @@ -226,25 +365,42 @@ get_all_gpus_json() { get_nvidia_gpus get_generic_gpus ) | jq -s ' - group_by(.description) | map({ - id: (.[0].description | ascii_downcase | gsub("[^a-z0-9]"; "-") | gsub("-+"; "-") | sub("^-"; "") | sub("-$"; "")), - description: .[0].description, - type: "gpu", - total: length, - init: { - deviceRequests: { - Driver: .[0].init.deviceRequests.Driver, - (if .[0].init.deviceRequests.Driver == "nvidia" then "DeviceIDs" else "Devices" end): (map(.init.deviceRequests.Devices[]?) | unique), - Capabilities: [["gpu"]] - }, - Binds: (map(.init.Binds[]?) | unique), - CapAdd: (map(.init.CapAdd[]?) | unique), - GroupAdd: (map(.init.GroupAdd[]?) | unique), - SecurityOpt: .[0].init.SecurityOpt, - ShmSize: .[0].init.ShmSize, - IpcMode: .[0].init.IpcMode + group_by(.description) | map( + { + id: (.[0].description | ascii_downcase | gsub("[^a-z0-9]"; "-") | gsub("-+"; "-") | sub("^-"; "") | sub("-$"; "")), + description: .[0].description, + type: "gpu", + total: length, + driverVersion: (.[0].driverVersion // null), + memoryTotal: (.[0].memoryTotal // null), + platform: (if .[0].init.deviceRequests.Driver == "amdgpu" then "amd" else .[0].init.deviceRequests.Driver end), + init: ( + if .[0].init.deviceRequests.Driver == "nvidia" then + { + deviceRequests: { + Driver: .[0].init.deviceRequests.Driver, + DeviceIDs: (map(.init.deviceRequests.Devices[]?) | unique), + Capabilities: [["gpu"]] + } + } + else + { + advanced: { + Driver: .[0].init.deviceRequests.Driver, + Devices: (map(.init.deviceRequests.Devices[]?) | unique), + Capabilities: [["gpu"]], + Binds: (map(.init.Binds[]?) | unique), + CapAdd: (map(.init.CapAdd[]?) | unique), + GroupAdd: (map(.init.GroupAdd[]?) | unique), + SecurityOpt: .[0].init.SecurityOpt, + ShmSize: .[0].init.ShmSize, + IpcMode: .[0].init.IpcMode + } | del(.. | select(. == null)) | del(.. | select(. == [])) + } + end + ) } | del(.. | select(. == null)) | del(.. | select(. == [])) - }) | map(if .init.deviceRequests.Driver == null then del(.init.deviceRequests.Driver) else . end) + ) ' } diff --git a/scripts/ocean-node-quickstart.sh b/scripts/ocean-node-quickstart.sh index 6e350a38e..5a6da078d 100755 --- a/scripts/ocean-node-quickstart.sh +++ b/scripts/ocean-node-quickstart.sh @@ -277,25 +277,127 @@ fi # Function to check for NVIDIA GPUs get_nvidia_gpus() { if command -v nvidia-smi &> /dev/null; then - nvidia-smi --query-gpu=name,uuid --format=csv,noheader | while IFS=, read -r name uuid; do + nvidia-smi --query-gpu=name,uuid,driver_version,memory.total --format=csv,noheader | while IFS=, read -r name uuid driver_version memory_total; do name=$(echo "$name" | xargs) uuid=$(echo "$uuid" | xargs) - jq -c -n \ + driver_version=$(echo "$driver_version" | xargs) + memory_total=$(echo "$memory_total" | xargs) + + jq -c -n \ --arg name "$name" \ --arg uuid "$uuid" \ + --arg driver_version "$driver_version" \ + --arg memory_total "$memory_total" \ '{ description: $name, + driverVersion: (if $driver_version != "" then $driver_version else null end), + memoryTotal: (if $memory_total != "" then $memory_total else null end), init: { deviceRequests: { Driver: "nvidia", Devices: [$uuid] } } - }' + } | del(.. | select(. == null))' done fi } +get_driver_version() { + local module="$1" + local ver + ver=$(modinfo "$module" 2>/dev/null | awk '/^version:/ {print $2; exit}') + if [ -n "$ver" ]; then + echo "$ver" + return + fi + + if modinfo "$module" 2>/dev/null | grep -q "^srcversion:"; then + uname -r + return + fi +} + +get_intel_driver_version() { + # oneAPI / Level Zero runtime (Intel Arc) + local ver + ver=$(clinfo 2>/dev/null | awk '/Driver Version/ {print $NF; exit}') + [ -n "$ver" ] && { echo "$ver"; return; } + ver=$(dpkg -l 2>/dev/null | awk '/intel-level-zero-gpu/ {print $3; exit}') + [ -z "$ver" ] && ver=$(rpm -q --qf '%{VERSION}' intel-level-zero-gpu 2>/dev/null) + [ -n "$ver" ] && { echo "$ver"; return; } + get_driver_version "xe" || get_driver_version "i915" +} + +get_amd_driver_version() { + local ver + ver=$(cat /opt/rocm/.info/version 2>/dev/null | head -n1) + [ -n "$ver" ] && { echo "ROCm $ver"; return; } + ver=$(rocm-smi --version 2>/dev/null | awk '/ROCm/ {print $NF; exit}') + [ -n "$ver" ] && { echo "$ver"; return; } + ver=$(dpkg -l 2>/dev/null | awk '/rocm-core/ {print $3; exit}') + [ -z "$ver" ] && ver=$(rpm -q --qf '%{VERSION}' rocm-core 2>/dev/null) + [ -n "$ver" ] && { echo "ROCm $ver"; return; } + get_driver_version "amdgpu" +} + +get_amd_vram() { + local card_path="$1" + local real_device_path + real_device_path=$(readlink -f "$card_path/device") + + local vram_bytes="" + for f in \ + "$real_device_path/mem_info_vram_total" \ + "$card_path/device/mem_info_vram_total"; do + if [ -r "$f" ]; then + vram_bytes=$(cat "$f" 2>/dev/null) + break + fi + done + + if [ -n "$vram_bytes" ] && [ "$vram_bytes" -gt 0 ] 2>/dev/null; then + echo $(( vram_bytes / 1024 / 1024 )) MiB + fi +} + +get_intel_vram() { + local card_path="$1" + local real_device_path + real_device_path=$(readlink -f "$card_path/device") + + for f in \ + "$real_device_path/drm/$(basename $card_path)/gt/gt0/mem_info_vram_total" \ + "$real_device_path/mem_info_vram_total"; do + if [ -r "$f" ]; then + local bytes + bytes=$(cat "$f" 2>/dev/null) + if [ -n "$bytes" ] && [ "$bytes" -gt 0 ] 2>/dev/null; then + echo "$(( bytes / 1024 / 1024 )) MiB" + return + fi + fi + done + + local slot + slot=$(basename "$real_device_path") + local raw + raw=$(lspci -s "$slot" -v 2>/dev/null | awk '/Memory at|prefetchable/ && /size=/ { + match($0, /size=([0-9]+[KMG])/, a); if (a[1]) { print a[1]; exit } + }') + + if [ -n "$raw" ]; then + local num unit + num=$(echo "$raw" | tr -d 'KMG') + unit=$(echo "$raw" | tr -d '0-9') + case "$unit" in + K) echo "$(( num / 1024 )) MiB" ;; + M) echo "${num} MiB" ;; + G) echo "$(( num * 1024 )) MiB" ;; + esac + fi +} + # Declare the associative array (hashmap) globally declare -A gpu_map @@ -309,40 +411,51 @@ map_pci_to_primary() { done } +# Function to check for other GPUs (AMD, Intel, etc.) via lspci +get_generic_gpus() { + if ! command -v lspci &> /dev/null; then + return + fi + + map_pci_to_primary + lspci -mm -n -d ::0300 | while read -r line; do process_pci_line "$line"; done + lspci -mm -n -d ::0302 | while read -r line; do process_pci_line "$line"; done +} + process_pci_line() { line="$1" - + slot=$(echo "$line" | awk '{print $1}') vendor_id_hex=$(echo "$line" | awk '{print $3}' | tr -d '"') - + if [[ "$vendor_id_hex" == "10de" ]] && command -v nvidia-smi &> /dev/null; then return fi - + full_info=$(lspci -s "$slot" -vmm) vendor_name=$(echo "$full_info" | grep "^Vendor:" | cut -f2-) device_name=$(echo "$full_info" | grep "^Device:" | cut -f2-) - + description="$vendor_name $device_name" - pci_id="0000:$slot" - + pci_id="0000:$slot" + driver="" - if [[ "$vendor_id_hex" == "1002" ]]; then + if [[ "$vendor_id_hex" == "1002" ]]; then # AMD driver="amdgpu" - elif [[ "$vendor_id_hex" = "8086" ]]; then + elif [[ "$vendor_id_hex" = "8086" ]]; then # Intel driver="intel" fi device_id="" card_path="" if [ -n "${gpu_map[$pci_id]}" ]; then - device_id="${gpu_map[$pci_id]}" + device_id="${gpu_map[$pci_id]}" # e.g. /dev/dri/card0 card_name=$(basename "$device_id") card_path="/sys/class/drm/$card_name" else device_id="${pci_id}" fi - + local devices=() local binds=() local cap_add=() @@ -355,38 +468,52 @@ process_pci_line() { local real_device_path=$(readlink -f "$card_path/device") local render_name="" if [ -d "$real_device_path/drm" ]; then - render_name=$(ls "$real_device_path/drm" | grep "^renderD" | head -n 1) + render_name=$(ls "$real_device_path/drm" | grep "^renderD" | head -n 1) fi case "$vendor_id_hex" in - "1002") # AMD + "1002") # AMD (0x1002) + local amd_driver_version + amd_driver_version=$(get_amd_driver_version) + local amd_memory_total + amd_memory_total=$(get_amd_vram "$card_path") + if [ -e "/dev/dxg" ]; then devices+=("/dev/dxg") else devices+=("/dev/kfd") fi [ -n "$render_name" ] && devices+=("/dev/dri/$render_name") - devices+=("$device_id") + devices+=("$device_id") # /dev/dri/cardX + [ -e "/opt/rocm/lib/libhsa-runtime64.so.1" ] && \ - binds+=("/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1") + binds+=("/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1") + cap_add+=("SYS_PTRACE") ipc_mode="\"host\"" shm_size="8589934592" security_opt='{"seccomp": "unconfined"}' ;; - "8086") # Intel - [ -n "$render_name" ] && devices+=("/dev/dri/$render_name") - devices+=("$device_id") - group_add+=("video" "render") - cap_add+=("SYS_ADMIN") + + "8086") # Intel (0x8086) + local intel_driver_version + intel_driver_version=$(get_intel_driver_version) + local intel_memory_total + intel_memory_total=$(get_intel_vram "$card_path") + + [ -n "$render_name" ] && devices+=("/dev/dri/$render_name") + devices+=("$device_id") + + group_add+=("video" "render") + cap_add+=("SYS_ADMIN") ;; esac else - if [[ "$vendor_id_hex" == "1002" ]] || [[ "$vendor_id_hex" == "8086" ]]; then - if [[ "$device_id" == /dev/* ]]; then - devices+=("$device_id") - fi - fi + if [[ "$vendor_id_hex" == "1002" ]] || [[ "$vendor_id_hex" == "8086" ]]; then + if [[ "$device_id" == /dev/* ]]; then + devices+=("$device_id") + fi + fi fi json_devices=$(printf '%s\n' "${devices[@]}" | jq -R . | jq -s . | jq 'map(select(length > 0))') @@ -395,13 +522,21 @@ process_pci_line() { json_group=$(printf '%s\n' "${group_add[@]}" | jq -R . | jq -s . | jq 'map(select(length > 0))') if [ "$(echo "$json_devices" | jq length)" -eq 0 ]; then - json_devices="[\"$device_id\"]" + json_devices="[\"$device_id\"]" fi + local driver_version="" + local memory_total="" + case "$vendor_id_hex" in + "1002") driver_version="$amd_driver_version"; memory_total="$amd_memory_total" ;; + "8086") driver_version="$intel_driver_version"; memory_total="$intel_memory_total" ;; + esac jq -c -n \ --arg desc "$description" \ --arg driver "$driver" \ --arg device_id "$device_id" \ + --arg driver_version "$driver_version" \ + --arg memory_total "$memory_total" \ --argjson dev "$json_devices" \ --argjson bind "$json_binds" \ --argjson cap "$json_cap" \ @@ -411,6 +546,8 @@ process_pci_line() { --argjson ipc "$ipc_mode" \ '{ description: $desc, + driverVersion: (if $driver_version != "" then $driver_version else null end), + memoryTotal: (if $memory_total != "" then $memory_total else null end), init: { deviceRequests: { Driver: (if $driver != "" then $driver else null end), @@ -427,41 +564,48 @@ process_pci_line() { } | del(.. | select(. == null)) | del(.. | select(. == []))' } -# Function to check for other GPUs (AMD, Intel, etc.) via lspci -get_generic_gpus() { - if ! command -v lspci &> /dev/null; then - return - fi - map_pci_to_primary - lspci -mm -n -d ::0300 | while read -r line; do process_pci_line "$line"; done - lspci -mm -n -d ::0302 | while read -r line; do process_pci_line "$line"; done -} - # Function to get all GPUs in JSON array format get_all_gpus_json() { ( get_nvidia_gpus get_generic_gpus ) | jq -s ' - group_by(.description) | map({ - id: (.[0].description | ascii_downcase | gsub("[^a-z0-9]"; "-") | gsub("-+"; "-") | sub("^-"; "") | sub("-$"; "")), - description: .[0].description, - type: "gpu", - total: length, - init: { - deviceRequests: { - Driver: .[0].init.deviceRequests.Driver, - (if .[0].init.deviceRequests.Driver == "nvidia" then "DeviceIDs" else "Devices" end): (map(.init.deviceRequests.Devices[]?) | unique), - Capabilities: [["gpu"]] - }, - Binds: (map(.init.Binds[]?) | unique), - CapAdd: (map(.init.CapAdd[]?) | unique), - GroupAdd: (map(.init.GroupAdd[]?) | unique), - SecurityOpt: .[0].init.SecurityOpt, - ShmSize: .[0].init.ShmSize, - IpcMode: .[0].init.IpcMode + group_by(.description) | map( + { + id: (.[0].description | ascii_downcase | gsub("[^a-z0-9]"; "-") | gsub("-+"; "-") | sub("^-"; "") | sub("-$"; "")), + description: .[0].description, + type: "gpu", + total: length, + driverVersion: (.[0].driverVersion // null), + memoryTotal: (.[0].memoryTotal // null), + platform: (if .[0].init.deviceRequests.Driver == "amdgpu" then "amd" else .[0].init.deviceRequests.Driver end), + init: ( + if .[0].init.deviceRequests.Driver == "nvidia" then + { + deviceRequests: { + Driver: .[0].init.deviceRequests.Driver, + DeviceIDs: (map(.init.deviceRequests.Devices[]?) | unique), + Capabilities: [["gpu"]] + } + } + else + { + advanced: { + Driver: .[0].init.deviceRequests.Driver, + Devices: (map(.init.deviceRequests.Devices[]?) | unique), + Capabilities: [["gpu"]], + Binds: (map(.init.Binds[]?) | unique), + CapAdd: (map(.init.CapAdd[]?) | unique), + GroupAdd: (map(.init.GroupAdd[]?) | unique), + SecurityOpt: .[0].init.SecurityOpt, + ShmSize: .[0].init.ShmSize, + IpcMode: .[0].init.IpcMode + } | del(.. | select(. == null)) | del(.. | select(. == [])) + } + end + ) } | del(.. | select(. == null)) | del(.. | select(. == [])) - }) | map(if .init.deviceRequests.Driver == null then del(.init.deviceRequests.Driver) else . end) + ) ' } diff --git a/src/@types/C2D/C2D.ts b/src/@types/C2D/C2D.ts index 5b52751fd..b154eb8ce 100644 --- a/src/@types/C2D/C2D.ts +++ b/src/@types/C2D/C2D.ts @@ -1,5 +1,5 @@ import { MetadataAlgorithm, ConsumerParameter } from '@oceanprotocol/ddo-js' -import type { BaseFileObject } from '../fileObject.js' +import type { BaseFileObject, StorageObject, EncryptMethod } from '../fileObject.js' export enum C2DClusterType { // eslint-disable-next-line no-unused-vars OPF_K8 = 0, @@ -56,6 +56,12 @@ export interface ComputeResource { min: number // min number of resource needed for a job max: number // max number of resource for a job inUse?: number // for display purposes + driverVersion?: string + memoryTotal?: string + /** + * `nvidia` | `amd` | `intel` + */ + platform?: string init?: dockerHwInit } export interface ComputeResourceRequest { @@ -188,16 +194,14 @@ export interface ComputeJob { queueMaxWaitTime: number // max time in seconds a job can wait in the queue before being started } +export interface ComputeOutputEncryption { + encryptMethod: EncryptMethod.AES // in future we will support more ciphers + key: string // AES symetric key +} + export interface ComputeOutput { - publishAlgorithmLog?: boolean - publishOutput?: boolean - providerAddress?: string - providerUri?: string - metadataUri?: string - nodeUri?: string - owner?: string - secretStoreUri?: string - whitelist?: string[] + remoteStorage?: StorageObject + encryption?: ComputeOutputEncryption } export interface ComputeAsset { @@ -266,6 +270,7 @@ export interface DBComputeJob extends ComputeJob { additionalViewers?: string[] // addresses of additional addresses that can get results algoDuration: number // duration of the job in seconds encryptedDockerRegistryAuth?: string + output?: string // this is always an ECIES encrypted string, that decodes to ComputeOutput interface } // make sure we keep them both in sync diff --git a/src/@types/KeyManager.ts b/src/@types/KeyManager.ts index f117ea02f..516d358fa 100644 --- a/src/@types/KeyManager.ts +++ b/src/@types/KeyManager.ts @@ -52,8 +52,9 @@ export interface IKeyProvider { * Encrypts data according to a given algorithm * @param data data to encrypt * @param algorithm encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) */ - encrypt(data: Uint8Array, algorithm: EncryptMethod): Promise + encrypt(data: Uint8Array, algorithm: EncryptMethod, key?: Uint8Array): Promise /** * Decrypts data according to a given algorithm using node keys @@ -65,9 +66,14 @@ export interface IKeyProvider { * Encrypts a stream according to a given algorithm using node keys * @param inputStream - Readable stream to encrypt * @param algorithm - Encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) * @returns Readable stream with encrypted data */ - encryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable + encryptStream( + inputStream: Readable, + algorithm: EncryptMethod, + key?: Uint8Array + ): Readable /** * Decrypts a stream according to a given algorithm using node keys * @param inputStream - Readable stream to decrypt diff --git a/src/@types/commands.ts b/src/@types/commands.ts index fb3cd5d37..395487b44 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -4,18 +4,10 @@ import { DDO } from '@oceanprotocol/ddo-js' import type { ComputeAsset, ComputeAlgorithm, - ComputeOutput, ComputeResourceRequest, DBComputeJobMetadata } from './C2D/C2D.js' -import { - ArweaveFileObject, - FileObjectType, - EncryptMethod, - IpfsFileObject, - UrlFileObject, - BaseFileObject -} from './fileObject' +import { FileObjectType, StorageObject, EncryptMethod } from './fileObject' export interface Command { command: string // command name @@ -69,7 +61,7 @@ export interface FileInfoCommand extends Command { did?: string serviceId?: string fileIndex?: number - file?: UrlFileObject | ArweaveFileObject | IpfsFileObject + file?: StorageObject checksum?: boolean } // group these 2 @@ -135,7 +127,7 @@ export interface EncryptFileCommand extends Command { consumerAddress: string signature: string encryptionType?: EncryptMethod.AES | EncryptMethod.ECIES - files?: BaseFileObject + files?: StorageObject rawData?: Buffer policyServer?: any // object to pass to policy server } @@ -228,6 +220,7 @@ export interface ComputeInitializeCommand extends Command { policyServer?: any // object to pass to policy server queueMaxWaitTime?: number // max time in seconds a job can wait in the queue before being started encryptedDockerRegistryAuth?: string + output?: string // this is always an ECIES encrypted string, that decodes to ComputeOutput interface } export interface FreeComputeStartCommand extends Command { @@ -237,7 +230,7 @@ export interface FreeComputeStartCommand extends Command { environment: string algorithm: ComputeAlgorithm datasets?: ComputeAsset[] - output?: ComputeOutput + output?: string // this is always an ECIES encrypted string, that decodes to ComputeOutput interface resources?: ComputeResourceRequest[] maxJobDuration?: number policyServer?: any // object to pass to policy server @@ -264,6 +257,7 @@ export interface ComputeGetResultCommand extends Command { nonce: string jobId: string index: number + offset?: number } export interface ComputeGetStreamableLogsCommand extends Command { consumerAddress: string diff --git a/src/@types/fileObject.ts b/src/@types/fileObject.ts index 4bfebfe2a..23803bc1f 100644 --- a/src/@types/fileObject.ts +++ b/src/@types/fileObject.ts @@ -44,11 +44,17 @@ export interface S3FileObject extends BaseFileObject { s3Access: S3Object } +export interface FtpFileObject extends BaseFileObject { + /** Full FTP or FTPS URL: ftp://[user:password@]host[:port]/path or ftps://... */ + url: string +} + export type StorageObject = | UrlFileObject | IpfsFileObject | ArweaveFileObject | S3FileObject + | FtpFileObject export interface StorageReadable { stream: Readable @@ -61,7 +67,8 @@ export enum FileObjectType { URL = 'url', IPFS = 'ipfs', ARWEAVE = 'arweave', - S3 = 's3' + S3 = 's3', + FTP = 'ftp' } export interface FileInfoRequest { @@ -80,11 +87,10 @@ export interface FileInfoResponse { encryptMethod?: EncryptMethod } -export interface FileInfoHttpRequest { - type?: FileObjectType - did?: string - hash?: string - url?: string - transactionId?: string - serviceId?: string -} +export type FileInfoHttpRequest = + | StorageObject + | { + did?: string + serviceId?: string + checksum?: boolean + } diff --git a/src/components/KeyManager/index.ts b/src/components/KeyManager/index.ts index 6b93552cd..407dffaf1 100644 --- a/src/components/KeyManager/index.ts +++ b/src/components/KeyManager/index.ts @@ -149,9 +149,14 @@ export class KeyManager { * This method encrypts data according to a given algorithm using node keys * @param data data to encrypt * @param algorithm encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) */ - async encrypt(data: Uint8Array, algorithm: EncryptMethod): Promise { - return await this.keyProvider.encrypt(data, algorithm) + async encrypt( + data: Uint8Array, + algorithm: EncryptMethod, + key?: Uint8Array + ): Promise { + return await this.keyProvider.encrypt(data, algorithm, key) } /** @@ -167,10 +172,15 @@ export class KeyManager { * Encrypts a stream according to a given algorithm using node keys * @param inputStream - Readable stream to encrypt * @param algorithm - Encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) * @returns Readable stream with encrypted data */ - encryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable { - return this.keyProvider.encryptStream(inputStream, algorithm) + encryptStream( + inputStream: Readable, + algorithm: EncryptMethod, + key?: Uint8Array + ): Readable { + return this.keyProvider.encryptStream(inputStream, algorithm, key) } /** diff --git a/src/components/KeyManager/providers/RawPrivateKeyProvider.ts b/src/components/KeyManager/providers/RawPrivateKeyProvider.ts index 5254690e0..ffa2837a2 100644 --- a/src/components/KeyManager/providers/RawPrivateKeyProvider.ts +++ b/src/components/KeyManager/providers/RawPrivateKeyProvider.ts @@ -88,22 +88,36 @@ export class RawPrivateKeyProvider implements IKeyProvider { * This method encrypts data according to a given algorithm using node keys * @param data data to encrypt * @param algorithm encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) */ // eslint-disable-next-line require-await - async encrypt(data: Uint8Array, algorithm: EncryptMethod): Promise { + async encrypt( + data: Uint8Array, + algorithm: EncryptMethod, + key?: Uint8Array + ): Promise { let encryptedData: Buffer const { privateKey, publicKey } = this if (algorithm === EncryptMethod.AES) { - // use first 16 bytes of public key as an initialisation vector + const cipherKey = + key !== undefined && key.length >= 32 + ? Buffer.from(key.subarray(0, 32)) + : privateKey.raw + if (key !== undefined && key.length < 32) { + throw new Error('encrypt: AES symmetric key must be at least 32 bytes') + } + if (cipherKey.length !== 32) { + throw new Error('encrypt: privateKey must be 32 bytes for AES-256') + } const initVector = publicKey.subarray(0, 16) - // creates cipher object, with the given algorithm, key and initialization vector - const cipher = crypto.createCipheriv('aes-256-cbc', privateKey.raw, initVector) - // encoding is ignored because we are working with bytes and want to return a buffer + const cipher = crypto.createCipheriv('aes-256-cbc', cipherKey, initVector) encryptedData = Buffer.concat([cipher.update(data), cipher.final()]) } else if (algorithm === EncryptMethod.ECIES) { - const sk = new eciesjs.PrivateKey(privateKey.raw) - // get public key from Elliptic curve - encryptedData = eciesjs.encrypt(sk.publicKey.toHex(), data) + const recipientPublicKeyHex = + key && key.length > 0 + ? Buffer.from(key).toString('hex') + : new eciesjs.PrivateKey(privateKey.raw).publicKey.toHex() + encryptedData = Buffer.from(eciesjs.encrypt(recipientPublicKeyHex, data)) } return encryptedData } @@ -128,7 +142,7 @@ export class RawPrivateKeyProvider implements IKeyProvider { decryptedData = Buffer.concat([decipher.update(data), decipher.final()]) } else if (algorithm === EncryptMethod.ECIES) { const sk = new eciesjs.PrivateKey(privateKey.raw) - decryptedData = eciesjs.decrypt(sk.secret, data) + decryptedData = Buffer.from(eciesjs.decrypt(sk.secret, data)) } return decryptedData } @@ -137,9 +151,14 @@ export class RawPrivateKeyProvider implements IKeyProvider { * Encrypts a stream according to a given algorithm using node keys * @param inputStream - Readable stream to encrypt * @param algorithm - Encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) * @returns Readable stream with encrypted data */ - encryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable { + encryptStream( + inputStream: Readable, + algorithm: EncryptMethod, + key?: Uint8Array + ): Readable { if (!inputStream || typeof inputStream.pipe !== 'function') { throw new Error('encryptStream: inputStream must be a readable stream') } @@ -147,23 +166,29 @@ export class RawPrivateKeyProvider implements IKeyProvider { const { privateKey, publicKey } = this if (algorithm === EncryptMethod.AES) { + const cipherKey = + key !== undefined && key.length >= 32 + ? Buffer.from(key.subarray(0, 32)) + : privateKey.raw + if (key !== undefined && key.length < 32) { + throw new Error('encryptStream: AES symmetric key must be at least 32 bytes') + } + if (cipherKey.length !== 32) { + throw new Error('encryptStream: privateKey must be 32 bytes for AES-256') + } if (publicKey.length < 16) { throw new Error( 'encryptStream: publicKey must be at least 16 bytes for AES initialization vector' ) } - if (privateKey.raw.length !== 32) { - throw new Error('encryptStream: privateKey must be 32 bytes for AES-256') - } - // Use first 16 bytes of public key as an initialization vector const initVector = publicKey.subarray(0, 16) - // Create cipher transform stream - const cipher = crypto.createCipheriv('aes-256-cbc', privateKey.raw, initVector) - - // Pipe input stream through cipher and return the encrypted stream + const cipher = crypto.createCipheriv('aes-256-cbc', cipherKey, initVector) return inputStream.pipe(cipher) } else if (algorithm === EncryptMethod.ECIES) { - // ECIES doesn't support streaming, so we need to collect all data first + const recipientPublicKeyHex = + key !== undefined && key.length > 0 + ? Buffer.from(key).toString('hex') + : new eciesjs.PrivateKey(privateKey.raw).publicKey.toHex() const chunks: Buffer[] = [] const collector = new Transform({ transform(chunk, encoding, callback) { @@ -191,8 +216,7 @@ export class RawPrivateKeyProvider implements IKeyProvider { new Error('encryptStream: no data to encrypt (empty stream)') ) } - const sk = new eciesjs.PrivateKey(privateKey.raw) - const encryptedData = eciesjs.encrypt(sk.publicKey.toHex(), data) + const encryptedData = eciesjs.encrypt(recipientPublicKeyHex, data) this.push(Buffer.from(encryptedData)) callback() } catch (err) { diff --git a/src/components/P2P/handleProtocolCommands.ts b/src/components/P2P/handleProtocolCommands.ts index 91b5abefb..0750dec33 100644 --- a/src/components/P2P/handleProtocolCommands.ts +++ b/src/components/P2P/handleProtocolCommands.ts @@ -12,6 +12,7 @@ import { checkGlobalConnectionsRateLimit, checkRequestsRateLimit } from '../../utils/validators.js' +import { lpStream } from '@libp2p/utils' import type { Connection, Stream } from '@libp2p/interface' export class ReadableString extends Readable { @@ -40,28 +41,54 @@ export async function handleProtocolCommands(stream: Stream, connection: Connect P2P_LOGGER.logMessage('Incoming connection from peer ' + remotePeer, true) P2P_LOGGER.logMessage('Using ' + remoteAddr, true) - const sendErrorAndClose = async (httpStatus: number, error: string) => { + stream.resume() + const lp = lpStream(stream) + const handshakeSignal = () => AbortSignal.timeout(30_000) + const dataWriteSignal = () => AbortSignal.timeout(30 * 60_000) + + const sendErrorAndClose = async ( + httpStatus: number, + error: string, + errorDebug?: Record + ) => { try { - // Check if stream is already closed if (stream.status === 'closed' || stream.status === 'closing') { P2P_LOGGER.warn('Stream already closed, cannot send error response') return } - - // Resume stream in case it's paused - we need to write - stream.resume() - const status = { httpStatus, error } - stream.send(uint8ArrayFromString(JSON.stringify(status))) + const status = errorDebug + ? { httpStatus, error, errorDebug } + : { httpStatus, error } + await lp.write(uint8ArrayFromString(JSON.stringify(status)), { + signal: handshakeSignal() + }) await stream.close() } catch (e) { - P2P_LOGGER.error(`Error sending error response: ${e.message}`) + const msg = e instanceof Error ? e.message : e != null ? String(e) : 'Unknown error' + P2P_LOGGER.error(`Error sending error response: ${msg}`) try { stream.abort(e as Error) } catch {} } } - // Rate limiting and deny list checks + // Read the command first so the client always gets a response after writing. + // Rate limiting checks happen after reading to maintain the write→read protocol order. + let task: Command + try { + const cmdBytes = await lp.read({ signal: handshakeSignal() }) + const str = uint8ArrayToString(cmdBytes.subarray()) + task = JSON.parse(str) as Command + } catch (err) { + P2P_LOGGER.log( + LOG_LEVELS_STR.LEVEL_ERROR, + `Unable to process P2P command: ${err?.message ?? err}` + ) + await sendErrorAndClose(400, 'Invalid command') + return + } + + // Rate limiting and deny list checks (after reading command) const configuration = await getConfiguration() const { denyList } = configuration @@ -90,30 +117,6 @@ export async function handleProtocolCommands(stream: Stream, connection: Connect return } - // Resume the stream. We can now write. - stream.resume() - - // v3 streams are AsyncIterable - let task: Command - try { - for await (const chunk of stream) { - try { - const str = uint8ArrayToString(chunk.subarray()) - task = JSON.parse(str) as Command - } catch (e) { - await sendErrorAndClose(400, 'Invalid command') - return - } - } - } catch (err) { - P2P_LOGGER.log( - LOG_LEVELS_STR.LEVEL_ERROR, - `Unable to process P2P command: ${err.message}` - ) - await sendErrorAndClose(400, 'Invalid command') - return - } - if (!task) { P2P_LOGGER.error('Invalid or missing task/command data!') await sendErrorAndClose(400, 'Invalid command') @@ -133,20 +136,16 @@ export async function handleProtocolCommands(stream: Stream, connection: Connect task.caller = remotePeer.toString() const response: P2PCommandResponse = await handler.handle(task) - // Send status first - stream.send(uint8ArrayFromString(JSON.stringify(response.status))) + // Send status first (length-prefixed) + await lp.write(uint8ArrayFromString(JSON.stringify(response.status)), { + signal: handshakeSignal() + }) - // Stream data chunks without buffering, with backpressure support + // Stream data chunks as length-prefixed messages if (response.stream) { for await (const chunk of response.stream as Readable) { const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) - - // Handle backpressure - if send returns false, wait for drain - if (!stream.send(bytes)) { - await stream.onDrain({ - signal: AbortSignal.timeout(30000) // 30 second timeout for drain - }) - } + await lp.write(bytes, { signal: dataWriteSignal() }) } } diff --git a/src/components/P2P/index.ts b/src/components/P2P/index.ts index 16a4785b0..c8861148a 100644 --- a/src/components/P2P/index.ts +++ b/src/components/P2P/index.ts @@ -5,7 +5,8 @@ import { handleProtocolCommands } from './handlers.js' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import type { Stream } from '@libp2p/interface' +import { LengthPrefixedStream, lpStream } from '@libp2p/utils' +import type { Connection, Stream } from '@libp2p/interface' import { bootstrap } from '@libp2p/bootstrap' import { noise } from '@chainsafe/libp2p-noise' @@ -721,14 +722,45 @@ export class OceanP2P extends EventEmitter { return null } + async send( + lp: LengthPrefixedStream, + message: string, + options: { signal: AbortSignal } + ) { + await lp.write(uint8ArrayFromString(message), { signal: options.signal }) + const statusBytes = await lp.read({ signal: options.signal }) + return { + status: JSON.parse(uint8ArrayToString(statusBytes.subarray())), + stream: { + [Symbol.asyncIterator]: async function* () { + try { + while (true) { + const chunk = await lp.read() + yield chunk.subarray ? chunk.subarray() : chunk + } + } catch {} + } + } + } + } + async sendTo( peerName: string, message: string, multiAddrs?: string[] ): Promise<{ status: any; stream?: AsyncIterable }> { - P2P_LOGGER.logMessage('SendTo() node ' + peerName + ' task: ' + message, true) + const options = { + signal: AbortSignal.timeout(10_000), + priority: 100, + runOnLimitedConnection: true + } + let connection: Connection + let stream: Stream let peerId + + P2P_LOGGER.logMessage('SendTo() node ' + peerName + ' task: ' + message, true) + try { peerId = peerIdFromString(peerName) } catch (e) { @@ -741,14 +773,9 @@ export class OceanP2P extends EventEmitter { return { status: { httpStatus: 404, error: 'Invalid peer' } } } - let multiaddrs: Multiaddr[] = [] - if (!multiAddrs || multiAddrs.length < 1) { - multiaddrs = await this.getPeerMultiaddrs(peerName) - } else { - for (const addr of multiAddrs) { - multiaddrs.push(multiaddr(addr)) - } - } + const multiaddrs = multiAddrs?.length + ? multiAddrs.map((addr) => multiaddr(addr)) + : await this.getPeerMultiaddrs(peerName) if (multiaddrs.length < 1) { const error = `Cannot find any address to dial for peer: ${peerId}` @@ -756,18 +783,12 @@ export class OceanP2P extends EventEmitter { return { status: { httpStatus: 404, error } } } - let stream: Stream try { - const options = { - signal: AbortSignal.timeout(10000), - priority: 100, - runOnLimitedConnection: true - } - const connection = await this._libp2p.dial(multiaddrs, options) + connection = await this._libp2p.dial(multiaddrs, options) if (connection.remotePeer.toString() !== peerId.toString()) { - const error = `Invalid peer on the other side: ${connection.remotePeer.toString()}` - P2P_LOGGER.error(error) - return { status: { httpStatus: 404, error } } + throw new Error( + `Invalid peer on the other side: ${connection.remotePeer.toString()}` + ) } stream = await connection.newStream(this._protocol, options) } catch (e) { @@ -776,33 +797,39 @@ export class OceanP2P extends EventEmitter { return { status: { httpStatus: 404, error } } } - if (!stream) { - return { status: { httpStatus: 404, error: 'Unable to get remote P2P stream' } } - } - + let streamErr: Error | null = null try { - // Send message and close write side - stream.send(uint8ArrayFromString(message)) - await stream.close() - - // Read and parse status from first chunk - const iterator = stream[Symbol.asyncIterator]() - const { done, value } = await iterator.next() + return await this.send(lpStream(stream), message, options) + } catch (err) { + try { + stream.abort(err as Error) + } catch {} + streamErr = err + } - if (done || !value) { - return { status: { httpStatus: 500, error: 'No response from peer' } } - } + // abortConnectionOnPingFailure is disabled to keep long-running download streams alive, + // so stale connections are not evicted automatically. On a stale stream error, close the + // connection and retry once so the next dial establishes a fresh one. + if (!streamErr.message.includes('closed') && !streamErr.message.includes('reset')) { + P2P_LOGGER.error(`P2P communication error: ${streamErr.message}`) + return { status: { httpStatus: 500, error: `P2P error: ${streamErr.message}` } } + } - const status = JSON.parse(uint8ArrayToString(value.subarray())) + P2P_LOGGER.warn(`Stale connection to ${peerId}, retrying: ${streamErr.message}`) + try { + await connection.close() + } catch {} + connection = await this._libp2p.dial(multiaddrs, options) + stream = await connection.newStream(this._protocol, options) - // Return status and remaining stream - return { status, stream: { [Symbol.asyncIterator]: () => iterator } } - } catch (err) { - P2P_LOGGER.error(`P2P communication error: ${err.message}`) + try { + return await this.send(lpStream(stream), message, options) + } catch (retryErr) { try { - stream.abort(err as Error) + stream.abort(retryErr as Error) } catch {} - return { status: { httpStatus: 500, error: `P2P error: ${err.message}` } } + P2P_LOGGER.error(`P2P communication error on retry: ${retryErr.message}`) + return { status: { httpStatus: 500, error: `P2P error: ${retryErr.message}` } } } } @@ -932,10 +959,11 @@ export class OceanP2P extends EventEmitter { const cid = await cidFromRawString(input) const peersFound = [] try { - // @ts-ignore ignore the type mismatch const f = this._libp2p.contentRouting.findProviders(cid, { - signal: AbortSignal.timeout(timeout || 20000) // 20 seconds // on timeout the query ends with an abort signal => CodeError: Query aborted - }) + queryFuncTimeout: timeout || 20000 // 20 seconds + // on timeout the query ends with an abort signal => CodeError: Query aborted + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any) for await (const value of f) { peersFound.push(value) } diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index 671eee947..2423090a9 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -5,7 +5,6 @@ import type { ComputeAlgorithm, ComputeAsset, ComputeJob, - ComputeOutput, ComputeResourceRequest, ComputeResourceRequestWithPrice, ComputeResourceType, @@ -84,7 +83,7 @@ export abstract class C2DEngine { public abstract startComputeJob( assets: ComputeAsset[], algorithm: ComputeAlgorithm, - output: ComputeOutput, + output: string, environment: string, owner: string, maxJobDuration: number, @@ -112,7 +111,8 @@ export abstract class C2DEngine { public abstract getComputeJobResult( consumerAddress: string, jobId: string, - index: number + index: number, + offset?: number ): Promise<{ stream: Readable; headers: any }> public abstract cleanupExpiredStorage(job: DBComputeJob): Promise diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index c43f19658..850161eeb 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -10,9 +10,9 @@ import type { C2DClusterInfo, ComputeEnvironment, ComputeAlgorithm, + ComputeOutput, ComputeAsset, ComputeJob, - ComputeOutput, DBComputeJob, DBComputeJobPayment, ComputeResult, @@ -361,13 +361,14 @@ export class C2DEngineDocker extends C2DEngine { } } - // get all jobs that are in the settle status - const jobs = await this.db.getJobs( - envs, - undefined, - undefined, + // get all jobs that needs to be paid + const jobs = await this.db.getJobsByStatus(envs, [ + C2DStatusNumber.AlgorithmFailed, + C2DStatusNumber.DiskQuotaExceeded, + C2DStatusNumber.ResultsFetchFailed, + C2DStatusNumber.ResultsUploadFailed, C2DStatusNumber.JobSettle - ) + ]) CORE_LOGGER.info(`ClaimPayments: Got ${jobs.length} jobs to check`) if (jobs.length > 0) { const providerAddress = this.getKeyManager().getEthAddress() @@ -439,10 +440,10 @@ export class C2DEngineDocker extends C2DEngine { continue } - // Calculate minimum duration - let minDuration = 0 - if (algoDuration < 0) minDuration += algoDuration * -1 - else minDuration += algoDuration + let minDuration = Math.abs(algoDuration) + if (minDuration > job.maxJobDuration) { + minDuration = job.maxJobDuration + } if ( `minJobDuration` in env && env.minJobDuration && @@ -1023,7 +1024,7 @@ export class C2DEngineDocker extends C2DEngine { public override async startComputeJob( assets: ComputeAsset[], algorithm: ComputeAlgorithm, - output: ComputeOutput, + output: string, environment: string, owner: string, maxJobDuration: number, @@ -1119,7 +1120,8 @@ export class C2DEngineDocker extends C2DEngine { terminationDetails: { exitCode: null, OOMKilled: null }, algoDuration: 0, queueMaxWaitTime: queueMaxWaitTime || 0, - encryptedDockerRegistryAuth // we store the encrypted docker registry auth in the job + encryptedDockerRegistryAuth, // we store the encrypted docker registry auth in the job + output } if (algorithm.meta.container && algorithm.meta.container.dockerfile) { @@ -1239,17 +1241,21 @@ export class C2DEngineDocker extends C2DEngine { } } catch (e) {} try { - const outputStat = statSync( - this.getC2DConfig().tempFolder + '/' + jobId + '/data/outputs/outputs.tar' - ) - if (outputStat) { - res.push({ - filename: 'outputs.tar', - filesize: outputStat.size, - type: 'output', - index - }) - index = index + 1 + // check if we have an output request. + const jobDb = await this.db.getJob(jobId) + if (jobDb.length < 1 || !jobDb[0].output) { + const outputStat = statSync( + this.getC2DConfig().tempFolder + '/' + jobId + '/data/outputs/outputs.tar' + ) + if (outputStat) { + res.push({ + filename: 'outputs.tar', + filesize: outputStat.size, + type: 'output', + index + }) + index = index + 1 + } } } catch (e) {} try { @@ -1294,7 +1300,8 @@ export class C2DEngineDocker extends C2DEngine { public override async getComputeJobResult( consumerAddress: string, jobId: string, - index: number + index: number, + offset: number = 0 ): Promise<{ stream: Readable; headers: any }> { const jobs = await this.db.getJob(jobId, null, null) if (jobs.length === 0 || jobs.length > 1) { @@ -1358,7 +1365,8 @@ export class C2DEngineDocker extends C2DEngine { if (i.type === 'output') { return { stream: createReadStream( - this.getC2DConfig().tempFolder + '/' + jobId + '/data/outputs/outputs.tar' + this.getC2DConfig().tempFolder + '/' + jobId + '/data/outputs/outputs.tar', + offset > 0 ? { start: offset } : undefined ), headers: { 'Content-Type': 'application/octet-stream' @@ -1837,15 +1845,59 @@ export class C2DEngineDocker extends C2DEngine { job.terminationDetails.OOMKilled = null job.terminationDetails.exitCode = null } - const outputsArchivePath = this.getC2DConfig().tempFolder + '/' + job.jobId + '/data/outputs/outputs.tar' + try { if (container) { - await pipeline( - await container.getArchive({ path: '/data/outputs' }), - createWriteStream(outputsArchivePath) - ) + // if we have an output request, stream to remote storage; otherwise write to local file + if (job.output) { + const decryptedOutput = await this.keyManager.decrypt( + Uint8Array.from(Buffer.from(job.output, 'hex')), + EncryptMethod.ECIES + ) + const output = JSON.parse(decryptedOutput.toString()) as ComputeOutput + const storage = Storage.getStorageClass( + output.remoteStorage, + await getConfiguration() + ) + + if ( + storage.hasUpload && + 'upload' in storage && + typeof storage.upload === 'function' + ) { + let uploadStream = (await container.getArchive({ + path: '/data/outputs' + })) as unknown as Readable + if (output.encryption && output.encryption?.key) { + const enc = output.encryption + const key = Uint8Array.from(Buffer.from(enc.key, 'hex')) + uploadStream = this.keyManager.encryptStream( + uploadStream, + enc.encryptMethod, + key + ) + } + const fname = + 'outputs-' + this.getC2DConfig().hash + '-' + job.jobId + '.tar' + await ( + storage as unknown as { + upload: (name: string, stream: Readable) => Promise + } + ).upload(fname, uploadStream) + } else { + await pipeline( + await container.getArchive({ path: '/data/outputs' }), + createWriteStream(outputsArchivePath) + ) + } + } else { + await pipeline( + await container.getArchive({ path: '/data/outputs' }), + createWriteStream(outputsArchivePath) + ) + } } } catch (e) { CORE_LOGGER.error('Failed to get outputs archive: ' + e.message) diff --git a/src/components/c2d/index.ts b/src/components/c2d/index.ts index 24c89fced..192f5a60f 100644 --- a/src/components/c2d/index.ts +++ b/src/components/c2d/index.ts @@ -43,7 +43,8 @@ export function omitDBComputeFieldsFromComputeJob(dbCompute: DBComputeJob): Comp 'isRunning', 'isStarted', 'containerImage', - 'encryptedDockerRegistryAuth' + 'encryptedDockerRegistryAuth', + 'output' ]) as ComputeJob return job } diff --git a/src/components/core/compute/getResults.ts b/src/components/core/compute/getResults.ts index 888d6f61d..6c1143457 100644 --- a/src/components/core/compute/getResults.ts +++ b/src/components/core/compute/getResults.ts @@ -65,7 +65,8 @@ export class ComputeGetResultHandler extends CommandHandler { const respStream = await engine.getComputeJobResult( task.consumerAddress, jobId, - task.index + task.index, + task.offset ?? 0 ) const response: P2PCommandResponse = { stream: respStream?.stream, diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index c3e40172b..6f965ce13 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -33,8 +33,12 @@ import { getAlgorithmImage } from '../../c2d/compute_engine_docker.js' import { Credentials, DDOManager } from '@oceanprotocol/ddo-js' import { checkCredentials } from '../../../utils/credentials.js' import { PolicyServer } from '../../policyServer/index.js' -import { generateUniqueID, getAlgoChecksums, validateAlgoForDataset } from './utils.js' - +import { + generateUniqueID, + getAlgoChecksums, + validateAlgoForDataset, + validateOutput +} from './utils.js' export class ComputeInitializeHandler extends CommandHandler { validate(command: ComputeInitializeCommand): ValidateParams { const validation = validateCommandParameters(command, [ @@ -208,6 +212,15 @@ export class ComputeInitializeHandler extends CommandHandler { } } + const isValidOutput = await validateOutput( + node, + task.output, + await getConfiguration() + ) + if (isValidOutput.status.httpStatus !== 200) { + return isValidOutput + } + // check algo let index = 0 const policyServer = new PolicyServer() @@ -234,10 +247,6 @@ export class ComputeInitializeHandler extends CommandHandler { credentials, metadata } = ddoInstance.getDDOFields() - CORE_LOGGER.logMessage( - `InitializeCompute: evaluating access for did=${ddoInstance.getDid()} serviceId=${elem.serviceId} metadataType=${metadata?.type} consumerAddress=${task.consumerAddress}`, - true - ) const isOrdable = isOrderingAllowedForAsset(ddo) if (!isOrdable.isOrdable) { CORE_LOGGER.error(isOrdable.reason) @@ -299,10 +308,6 @@ export class ComputeInitializeHandler extends CommandHandler { // check credentials (DDO level) let accessGrantedDDOLevel: boolean if (credentials) { - CORE_LOGGER.logMessage( - `InitializeCompute: DDO-level credentials found for did=${ddoInstance.getDid()} policyServerConfigured=${isPolicyServerConfigured()}`, - true - ) // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. if (isPolicyServerConfigured()) { @@ -313,10 +318,6 @@ export class ComputeInitializeHandler extends CommandHandler { task.consumerAddress, task.policyServer ) - CORE_LOGGER.logMessage( - `InitializeCompute: policy server DDO-level decision for did=${ddoInstance.getDid()} serviceId=${elem.serviceId} success=${response.success} httpStatus=${response.httpStatus} message=${response.message}`, - true - ) accessGrantedDDOLevel = response.success } else { accessGrantedDDOLevel = await checkCredentials( @@ -324,10 +325,6 @@ export class ComputeInitializeHandler extends CommandHandler { credentials as Credentials, await blockchain.getSigner() ) - CORE_LOGGER.logMessage( - `InitializeCompute: local DDO-level credentials decision for did=${ddoInstance.getDid()} serviceId=${elem.serviceId} success=${accessGrantedDDOLevel}`, - true - ) } if (!accessGrantedDDOLevel) { CORE_LOGGER.logMessage( @@ -357,10 +354,6 @@ export class ComputeInitializeHandler extends CommandHandler { // check credentials on service level // if using a policy server and we are here it means that access was granted (they are merged/assessed together) if (service.credentials) { - CORE_LOGGER.logMessage( - `InitializeCompute: service-level credentials found for did=${ddoInstance.getDid()} serviceId=${service.id} policyServerConfigured=${isPolicyServerConfigured()}`, - true - ) let accessGrantedServiceLevel: boolean if (isPolicyServerConfigured()) { // we use the previous check or we do it again @@ -372,10 +365,6 @@ export class ComputeInitializeHandler extends CommandHandler { task.consumerAddress, task.policyServer ) - CORE_LOGGER.logMessage( - `InitializeCompute: policy server service-level decision for did=${ddoInstance.getDid()} serviceId=${service.id} success=${response.success} httpStatus=${response.httpStatus} message=${response.message}`, - true - ) accessGrantedServiceLevel = accessGrantedDDOLevel || response.success } else { accessGrantedServiceLevel = await checkCredentials( @@ -383,10 +372,6 @@ export class ComputeInitializeHandler extends CommandHandler { service.credentials, await blockchain.getSigner() ) - CORE_LOGGER.logMessage( - `InitializeCompute: local service-level credentials decision for did=${ddoInstance.getDid()} serviceId=${service.id} success=${accessGrantedServiceLevel}`, - true - ) } if (!accessGrantedServiceLevel) { diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index e95795091..7cfc5d954 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -7,7 +7,12 @@ import { } from '../../../@types/commands.js' import { CommandHandler } from '../handler/handler.js' import { OceanNode } from '../../../OceanNode.js' -import { generateUniqueID, getAlgoChecksums, validateAlgoForDataset } from './utils.js' +import { + generateUniqueID, + getAlgoChecksums, + validateAlgoForDataset, + validateOutput +} from './utils.js' import { ValidateParams, buildInvalidRequestMessage, @@ -39,7 +44,20 @@ import { PolicyServer } from '../../policyServer/index.js' import { checkCredentials } from '../../../utils/credentials.js' import { checkAddressOnAccessList } from '../../../utils/accessList.js' -export class PaidComputeStartHandler extends CommandHandler { +export class CommonComputeHandler extends CommandHandler { + validate(command: PaidComputeStartCommand): ValidateParams { + return { + valid: true + } + } + + // eslint-disable-next-line require-await + async handle(task: PaidComputeStartCommand): Promise { + return null + } +} + +export class PaidComputeStartHandler extends CommonComputeHandler { validate(command: PaidComputeStartCommand): ValidateParams { const commandValidation = validateCommandParameters(command, [ 'environment', @@ -79,9 +97,8 @@ export class PaidComputeStartHandler extends CommandHandler { if (authValidationResponse.status.httpStatus !== 200) { return authValidationResponse } - + const node = this.getOceanNode() try { - const node = this.getOceanNode() // split compute env (which is already in hash-envId format) and get the hash // then get env which might contain dashes as well const eIndex = task.environment.indexOf('-') @@ -564,6 +581,14 @@ export class PaidComputeStartHandler extends CommandHandler { } } } + const isValidOutput = await validateOutput( + node, + task.output, + await getConfiguration() + ) + if (isValidOutput.status.httpStatus !== 200) { + return isValidOutput + } try { const response = await engine.startComputeJob( task.datasets, @@ -634,7 +659,7 @@ export class PaidComputeStartHandler extends CommandHandler { } } -export class FreeComputeStartHandler extends CommandHandler { +export class FreeComputeStartHandler extends CommonComputeHandler { validate(command: FreeComputeStartCommand): ValidateParams { const commandValidation = validateCommandParameters(command, [ 'algorithm', @@ -713,6 +738,15 @@ export class FreeComputeStartHandler extends CommandHandler { } } } + const node = this.getOceanNode() + const isValidOutput = await validateOutput( + node, + task.output, + await getConfiguration() + ) + if (isValidOutput.status.httpStatus !== 200) { + return isValidOutput + } const policyServer = new PolicyServer() for (const elem of [...[task.algorithm], ...task.datasets]) { if (!('documentId' in elem)) { diff --git a/src/components/core/compute/utils.ts b/src/components/core/compute/utils.ts index dcbb4541d..b4f99a5e3 100644 --- a/src/components/core/compute/utils.ts +++ b/src/components/core/compute/utils.ts @@ -1,20 +1,17 @@ import { OceanNode } from '../../../OceanNode.js' -import { AlgoChecksums } from '../../../@types/C2D/C2D.js' +import { AlgoChecksums, ComputeOutput } from '../../../@types/C2D/C2D.js' import { OceanNodeConfig } from '../../../@types/OceanNode.js' -import { - ArweaveFileObject, - IpfsFileObject, - UrlFileObject -} from '../../../@types/fileObject.js' +import { StorageObject, EncryptMethod } from '../../../@types/fileObject.js' import { getFile } from '../../../utils/file.js' -import urlJoin from 'url-join' -import { fetchFileMetadata } from '../../../utils/asset.js' +import { Storage } from '../../storage/index.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' import { createHash } from 'crypto' import { FindDdoHandler } from '../../core/handler/ddoHandler.js' import { DDOManager, VersionedDDO } from '@oceanprotocol/ddo-js' +import { P2PCommandResponse } from '../../../@types/index.js' + export function generateUniqueID(jobStructure: any): string { const timestamp = BigInt(Date.now()) * 1_000_000n + (process.hrtime.bigint() % 1_000_000n) @@ -44,18 +41,12 @@ export async function getAlgoChecksums( } const fileArray = await getFile(algoDDO, algoServiceId, oceanNode) for (const file of fileArray) { - const url = - file.type === 'url' - ? (file as UrlFileObject).url - : file.type === 'arweave' - ? urlJoin(config.arweaveGateway, (file as ArweaveFileObject).transactionId) - : file.type === 'ipfs' - ? urlJoin(config.ipfsGateway, (file as IpfsFileObject).hash) - : null - const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined - - const { contentChecksum } = await fetchFileMetadata(url, 'get', false, headers) - checksums.files = checksums.files.concat(contentChecksum) + const storage = Storage.getStorageClass(file as StorageObject, config) + const fileInfo = await storage.fetchSpecificFileMetadata( + file as StorageObject, + true // force checksum + ) + checksums.files = checksums.files.concat(fileInfo.checksum) } const ddoInstance = DDOManager.getDDOClass(algoDDO) @@ -152,3 +143,109 @@ export async function validateAlgoForDataset( return false } } + +// checks if the encrypted string sent by the user is a valid ComputeOutput object +export async function validateOutput( + node: OceanNode, + output: string, + config: OceanNodeConfig +): Promise { + // null output is valid, because it's optional + if (!output) { + return { + status: { + httpStatus: 200, + error: null, + headers: null + }, + stream: null + } + } + + try { + const decrypted = await node + .getKeyManager() + .decrypt(Buffer.from(output, 'hex'), EncryptMethod.ECIES) + + const obj = JSON.parse(decrypted.toString()) as ComputeOutput + const storage = Storage.getStorageClass(obj.remoteStorage, config) + + const hasUploadSupport = + storage.hasUpload && 'upload' in storage && typeof storage.upload === 'function' + + // Only validate output-encryption semantics if backend can actually upload results. + if (!hasUploadSupport) { + return { + status: { + httpStatus: 400, + error: `Storage class has no support for upload`, + headers: null + }, + stream: null + } + } + const [isValidStorage, storageValidationError] = storage.validate() + if (!isValidStorage) { + return { + status: { + httpStatus: 400, + error: storageValidationError || 'Invalid remote storage configuration', + headers: null + }, + stream: null + } + } + if (obj.encryption && !obj.encryption.key) { + return { + status: { + httpStatus: 400, + error: `Encryption required, but no key`, + headers: null + }, + stream: null + } + } + if (obj.encryption && obj.encryption.encryptMethod !== EncryptMethod.AES) { + return { + status: { + httpStatus: 400, + error: `Only AES encryption is supported`, + headers: null + }, + stream: null + } + } + if (obj.encryption?.key) { + const keyBytes = Buffer.from(obj.encryption.key, 'hex') + if (keyBytes.length < 32) { + return { + status: { + httpStatus: 400, + error: `AES key must be at least 32 bytes (64 hex chars), got ${keyBytes.length} bytes`, + headers: null + }, + stream: null + } + } + } + + return { + status: { + httpStatus: 200, + error: null, + headers: null + }, + stream: null + } + } catch (e) { + const message = e instanceof Error ? e.message : String(e) + return { + status: { + httpStatus: 400, + error: `Invalid output: ${message}`, + headers: null + }, + stream: null + } + } +} diff --git a/src/components/core/handler/fileInfoHandler.ts b/src/components/core/handler/fileInfoHandler.ts index 68c5db08f..d11df081b 100644 --- a/src/components/core/handler/fileInfoHandler.ts +++ b/src/components/core/handler/fileInfoHandler.ts @@ -1,18 +1,12 @@ import { Readable } from 'stream' -import urlJoin from 'url-join' import { P2PCommandResponse } from '../../../@types/index.js' -import { - ArweaveFileObject, - IpfsFileObject, - UrlFileObject -} from '../../../@types/fileObject.js' +import { FileObjectType, StorageObject } from '../../../@types/fileObject.js' import { OceanNodeConfig } from '../../../@types/OceanNode.js' import { FileInfoCommand } from '../../../@types/commands.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' import { Storage } from '../../storage/index.js' import { CommandHandler } from './handler.js' import { validateDDOIdentifier } from './ddoHandler.js' -import { fetchFileMetadata } from '../../../utils/asset.js' import { ValidateParams, buildInvalidRequestMessage, @@ -22,34 +16,22 @@ import { getFile } from '../../../utils/file.js' import { getConfiguration } from '../../../utils/index.js' async function formatMetadata( - file: ArweaveFileObject | IpfsFileObject | UrlFileObject, + file: StorageObject, config: OceanNodeConfig -) { - const url = - file.type === 'url' - ? (file as UrlFileObject).url - : file.type === 'arweave' - ? urlJoin(config.arweaveGateway, (file as ArweaveFileObject).transactionId) - : file.type === 'ipfs' - ? urlJoin(config.ipfsGateway, (file as IpfsFileObject).hash) - : null - const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined - - const { contentLength, contentType, contentChecksum } = await fetchFileMetadata( - url, - 'get', - false, - headers +): Promise<{ + valid: boolean + contentLength: string + contentType: string + checksum?: string + name: string + type: string +}> { + const storage = Storage.getStorageClass(file, config) + const fileInfo = await storage.fetchSpecificFileMetadata(file, false) + CORE_LOGGER.logMessage( + `Metadata for file: ${fileInfo.contentLength} ${fileInfo.contentType}` ) - - return { - valid: true, - contentLength, - contentType, - checksum: contentChecksum, - name: new URL(url).pathname.split('/').pop() || '', - type: file.type - } + return fileInfo } export class FileInfoHandler extends CommandHandler { validate(command: FileInfoCommand): ValidateParams { @@ -72,6 +54,17 @@ export class FileInfoHandler extends CommandHandler { return buildInvalidRequestMessage('Invalid Request: no fields are present!') } } + + const matchesRegex = (value: string, regex: RegExp): boolean => regex.test(value) + if (command.did && !matchesRegex(command.did, /^did:op/)) { + return buildInvalidRequestMessage('Invalid Request: invalid did!') + } + if (command.type && !Object.values(FileObjectType).includes(command.type)) { + return buildInvalidRequestMessage( + 'Invalid Request: type must be one of ' + Object.values(FileObjectType).join(', ') + ) + } + return validation } diff --git a/src/components/core/handler/getJobs.ts b/src/components/core/handler/getJobs.ts index e50f5c4b1..a72a25add 100644 --- a/src/components/core/handler/getJobs.ts +++ b/src/components/core/handler/getJobs.ts @@ -32,8 +32,18 @@ export class GetJobsHandler extends CommandHandler { task.fromTimestamp, task.consumerAddrs ) + const sanitizedJobs = jobs.map((job) => { + if (job.algorithm) { + const { envs, meta, ...restAlgo } = job.algorithm + const sanitizedAlgo = meta + ? { ...restAlgo, meta: (({ rawcode, ...restMeta }) => restMeta)(meta) } + : restAlgo + return { ...job, algorithm: sanitizedAlgo } + } + return job + }) return { - stream: Readable.from(JSON.stringify(jobs)), + stream: Readable.from(JSON.stringify(sanitizedJobs)), status: { httpStatus: 200, error: null diff --git a/src/components/database/C2DDatabase.ts b/src/components/database/C2DDatabase.ts index dee7c57cf..87146e576 100755 --- a/src/components/database/C2DDatabase.ts +++ b/src/components/database/C2DDatabase.ts @@ -89,6 +89,13 @@ export class C2DDatabase extends AbstractDatabase { return await this.provider.getJobs(environments, fromTimestamp, consumerAddrs, status) } + async getJobsByStatus( + environments: string[], + status: C2DStatusNumber[] + ): Promise { + return await this.provider.getJobsByStatus(environments, status) + } + async updateImage(image: string): Promise { return await this.provider.updateImage(image) } diff --git a/src/components/database/ElasticSearchDatabase.ts b/src/components/database/ElasticSearchDatabase.ts index 65de69fec..e7d1df92b 100644 --- a/src/components/database/ElasticSearchDatabase.ts +++ b/src/components/database/ElasticSearchDatabase.ts @@ -11,8 +11,9 @@ import { OceanNodeDBConfig } from '../../@types' import { ElasticsearchSchema } from './ElasticSchemas.js' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../utils/logging/Logger.js' -import { DDOManager } from '@oceanprotocol/ddo-js' + import { validateDDO } from '../../utils/asset.js' +import { DDOManager } from '@oceanprotocol/ddo-js' export class ElasticsearchIndexerDatabase extends AbstractIndexerDatabase { private client: Client diff --git a/src/components/database/TypesenseDatabase.ts b/src/components/database/TypesenseDatabase.ts index 2e5645ea0..4129fdf02 100644 --- a/src/components/database/TypesenseDatabase.ts +++ b/src/components/database/TypesenseDatabase.ts @@ -13,8 +13,8 @@ import { AbstractLogDatabase, AbstractOrderDatabase } from './BaseDatabase.js' -import { DDOManager } from '@oceanprotocol/ddo-js' import { validateDDO } from '../../utils/asset.js' +import { DDOManager } from '@oceanprotocol/ddo-js' export class TypesenseOrderDatabase extends AbstractOrderDatabase { private provider: Typesense diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts index 9d87bd16f..d3796b7ce 100644 --- a/src/components/database/sqliteCompute.ts +++ b/src/components/database/sqliteCompute.ts @@ -45,7 +45,8 @@ function getInternalStructure(job: DBComputeJob): any { terminationDetails: job.terminationDetails, payment: job.payment, algoDuration: job.algoDuration, - queueMaxWaitTime: job.queueMaxWaitTime + queueMaxWaitTime: job.queueMaxWaitTime, + output: job.output } return internalBlob } @@ -438,7 +439,7 @@ export class SQLiteCompute implements ComputeDatabaseProvider { }) } - getJobs( + async getJobs( environments?: string[], fromTimestamp?: string, consumerAddrs?: string[], @@ -475,7 +476,44 @@ export class SQLiteCompute implements ComputeDatabaseProvider { selectSQL += ` WHERE ${conditions.join(' AND ')}` } selectSQL += ` ORDER BY dateCreated DESC` + return await this.doQuery(selectSQL, params, environments) + } + + async getJobsByStatus( + environments: string[], + status: C2DStatusNumber[] + ): Promise { + let selectSQL = `SELECT * FROM ${this.schema.name}` + + // sqlite3 bindings accept both strings and numbers; `status` is a numeric enum. + const params: Array = [] + const conditions: string[] = [] + + if (environments && environments.length > 0) { + const placeholders = environments.map(() => '?').join(',') + conditions.push(`environment IN (${placeholders})`) + params.push(...environments) + } + + if (status && status.length > 0) { + const placeholders = status.map(() => '?').join(',') + conditions.push(`status IN (${placeholders})`) + params.push(...status) + } + + if (conditions.length > 0) { + selectSQL += ` WHERE ${conditions.join(' AND ')}` + } + selectSQL += ` ORDER BY dateCreated DESC` + + return await this.doQuery(selectSQL, params, environments) + } + private doQuery( + selectSQL: string, + params: Array, + environments: string[] + ) { return new Promise((resolve, reject) => { this.db.all(selectSQL, params, (err, rows: any[] | undefined) => { if (err) { diff --git a/src/components/httpRoutes/commands.ts b/src/components/httpRoutes/commands.ts index 746d4caf7..779340638 100644 --- a/src/components/httpRoutes/commands.ts +++ b/src/components/httpRoutes/commands.ts @@ -135,7 +135,10 @@ directCommandRoute.post( } } catch (err) { HTTP_LOGGER.error(err.message) - res.status(500).send(err.message) + closedResponse = true + if (!res.headersSent) { + res.status(500).send(err.message) + } } } ) diff --git a/src/components/httpRoutes/compute.ts b/src/components/httpRoutes/compute.ts index 1515d441e..3411253d0 100644 --- a/src/components/httpRoutes/compute.ts +++ b/src/components/httpRoutes/compute.ts @@ -12,7 +12,6 @@ import { import type { ComputeAlgorithm, ComputeAsset, - ComputeOutput, ComputeResourceRequest } from '../../@types/C2D/C2D.js' import type { @@ -88,7 +87,7 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/compute`, async (req, res) => { (req.body.encryptedDockerRegistryAuth as string) || null } if (req.body.output) { - startComputeTask.output = req.body.output as ComputeOutput + startComputeTask.output = req.body.output } const response = await new PaidComputeStartHandler(req.oceanNode).handle( @@ -137,7 +136,7 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/freeCompute`, async (req, res) => (req.body.encryptedDockerRegistryAuth as string) || null } if (req.body.output) { - startComputeTask.output = req.body.output as ComputeOutput + startComputeTask.output = req.body.output } const response = await new FreeComputeStartHandler(req.oceanNode).handle( diff --git a/src/components/httpRoutes/fileInfo.ts b/src/components/httpRoutes/fileInfo.ts index 14f96f2a1..85925b352 100644 --- a/src/components/httpRoutes/fileInfo.ts +++ b/src/components/httpRoutes/fileInfo.ts @@ -1,11 +1,9 @@ import express, { Request, Response } from 'express' import { - ArweaveFileObject, FileInfoHttpRequest, FileObjectType, - IpfsFileObject, - UrlFileObject -} from '../../@types/fileObject' + StorageObject +} from '../../@types/fileObject.js' import { PROTOCOL_COMMANDS, SERVICES_API_BASE_PATH } from '../../utils/constants.js' import { FileInfoHandler } from '../core/handler/fileInfoHandler.js' import { HTTP_LOGGER } from '../../utils/logging/common.js' @@ -15,21 +13,6 @@ export const fileInfoRoute = express.Router() fileInfoRoute.use(express.json()) // Ensure JSON parsing middleware is used // Validation function -const validateFileInfoRequest = (req: FileInfoHttpRequest): boolean => { - // Helper function to check if a string matches a regular expression - const matchesRegex = (value: string, regex: RegExp): boolean => regex.test(value) - - if (!req.type && !req.did) return false // either 'type' or 'did' is required - if (req.type && !['ipfs', 'url', 'arweave'].includes(req.type)) return false // 'type' must be one of the allowed values - if (req.did && !matchesRegex(req.did, /^did:op/)) return false // 'did' must match the regex - if (req.type === 'ipfs' && !req.hash) return false // 'hash' is required if 'type' is 'ipfs' - if (req.type === 'url' && !req.url) return false // 'url' is required if 'type' is 'url' - if (req.type === 'arweave' && !req.transactionId) return false // 'transactionId' is required if 'type' is 'arweave' - if (!req.type && !req.serviceId) return false // 'serviceId' is required if 'type' is not provided - - return true -} - fileInfoRoute.post( `${SERVICES_API_BASE_PATH}/fileInfo`, express.urlencoded({ extended: true, type: '*/*' }), @@ -37,53 +20,33 @@ fileInfoRoute.post( const fileInfoReq: FileInfoHttpRequest = req.body as unknown as FileInfoHttpRequest HTTP_LOGGER.logMessage(`FileInfo request received: ${JSON.stringify(req.body)}`, true) - if (!validateFileInfoRequest(fileInfoReq)) { - res.status(400).send('Invalid request parameters') - return - } - try { + const hasType = + 'type' in fileInfoReq && + fileInfoReq.type != null && + String(fileInfoReq.type).trim() !== '' + const hasDid = + 'did' in fileInfoReq && + fileInfoReq.did != null && + String(fileInfoReq.did).trim() !== '' + if (!hasType && !hasDid) { + res.status(400).send('Invalid request parameters') + return + } // Retrieve the file info - let fileObject: UrlFileObject | IpfsFileObject | ArweaveFileObject + let fileObject: StorageObject let fileInfoTask: FileInfoCommand - if (fileInfoReq.did && fileInfoReq.serviceId) { + if (`did` in fileInfoReq && fileInfoReq.did && fileInfoReq.serviceId) { fileInfoTask = { command: PROTOCOL_COMMANDS.FILE_INFO, did: fileInfoReq.did, serviceId: fileInfoReq.serviceId, caller: req.caller } - } else if (fileInfoReq.type === 'url' && fileInfoReq.url) { - fileObject = { - type: 'url', - url: fileInfoReq.url, - method: 'GET' - } as UrlFileObject - fileInfoTask = { - command: PROTOCOL_COMMANDS.FILE_INFO, - file: fileObject, - type: fileObject.type as FileObjectType, - caller: req.caller - } - } else if (fileInfoReq.type === 'ipfs' && fileInfoReq.hash) { - fileObject = { - type: 'ipfs', - hash: fileInfoReq.hash, - method: 'GET' - } as IpfsFileObject - fileInfoTask = { - command: PROTOCOL_COMMANDS.FILE_INFO, - file: fileObject, - type: fileObject.type as FileObjectType, - caller: req.caller - } - } else if (fileInfoReq.type === 'arweave' && fileInfoReq.transactionId) { - fileObject = { - type: 'arweave', - transactionId: fileInfoReq.transactionId, - method: 'GET' - } as ArweaveFileObject + } else { + fileObject = { ...fileInfoReq } as StorageObject + fileInfoTask = { command: PROTOCOL_COMMANDS.FILE_INFO, file: fileObject, @@ -91,6 +54,7 @@ fileInfoRoute.post( caller: req.caller } } + const response = await new FileInfoHandler(req.oceanNode).handle(fileInfoTask) if (response.stream) { res.status(response.status.httpStatus) diff --git a/src/components/httpRoutes/provider.ts b/src/components/httpRoutes/provider.ts index f44a05744..faef472d1 100644 --- a/src/components/httpRoutes/provider.ts +++ b/src/components/httpRoutes/provider.ts @@ -10,7 +10,7 @@ import { DecryptDdoHandler } from '../core/handler/ddoHandler.js' import { DownloadHandler } from '../core/handler/downloadHandler.js' import { DownloadCommand } from '../../@types/commands.js' import { FeesHandler } from '../core/handler/feesHandler.js' -import { BaseFileObject, EncryptMethod } from '../../@types/fileObject.js' +import { StorageObject, EncryptMethod } from '../../@types/fileObject.js' import { P2PCommandResponse } from '../../@types/OceanNode.js' import { getEncryptMethodFromString } from '../../utils/crypt.js' @@ -119,7 +119,7 @@ providerRoutes.post(`${SERVICES_API_BASE_PATH}/encryptFile`, async (req, res) => if (req.is('application/json')) { // body as fileObject result = await new EncryptFileHandler(req.oceanNode).handle({ - files: req.body as BaseFileObject, + files: req.body as StorageObject, encryptionType: encryptMethod, command: PROTOCOL_COMMANDS.ENCRYPT_FILE, caller: req.caller, diff --git a/src/components/storage/FTPStorage.ts b/src/components/storage/FTPStorage.ts new file mode 100644 index 000000000..28f46d412 --- /dev/null +++ b/src/components/storage/FTPStorage.ts @@ -0,0 +1,188 @@ +import { Readable, PassThrough } from 'stream' +import { Client as FtpClient } from 'basic-ftp' +import { + FileInfoResponse, + FtpFileObject, + StorageReadable +} from '../../@types/fileObject.js' +import { OceanNodeConfig } from '../../@types/OceanNode.js' +import { Storage } from './Storage.js' + +const DEFAULT_FTP_PORT = 21 +const DEFAULT_FTPS_PORT = 990 + +function parseFtpUrl(url: string): { + host: string + port: number + user: string + password: string + path: string + secure: boolean +} { + const parsed = new URL(url) + if (parsed.protocol !== 'ftp:' && parsed.protocol !== 'ftps:') { + throw new Error(`Invalid FTP URL protocol: ${parsed.protocol}`) + } + const secure = parsed.protocol === 'ftps:' + const port = parsed.port + ? parseInt(parsed.port, 10) + : secure + ? DEFAULT_FTPS_PORT + : DEFAULT_FTP_PORT + const path = parsed.pathname && parsed.pathname !== '/' ? parsed.pathname : '' + return { + host: parsed.hostname, + port, + user: decodeURIComponent(parsed.username || 'anonymous'), + password: decodeURIComponent(parsed.password || 'anonymous@'), + path, + secure + } +} + +export class FTPStorage extends Storage { + public constructor(file: FtpFileObject, config: OceanNodeConfig) { + super(file, config, true) + const [isValid, message] = this.validate() + if (isValid === false) { + throw new Error(`Error validating the FTP file: ${message}`) + } + } + + async getReadableStream(): Promise { + const file = this.getFile() as FtpFileObject + const { host, port, user, password, path, secure } = parseFtpUrl(file.url) + const client = new FtpClient(30000) + const passThrough = new PassThrough() + + try { + await client.access({ + host, + port, + user, + password, + secure + }) + client.downloadTo(passThrough, path).then( + () => { + client.close() + }, + (err) => { + passThrough.destroy(err) + client.close() + } + ) + } catch (err) { + client.close() + throw err + } + + return { + httpStatus: 200, + stream: passThrough, + headers: {} + } + } + + /** + * Upload a file via FTP STOR. Appends filename to path if url ends with /. + */ + async upload( + filename: string, + stream: Readable + ): Promise<{ httpStatus: number; headers?: Record }> { + const file = this.getFile() as FtpFileObject + let { host, port, user, password, path, secure } = parseFtpUrl(file.url) + if (path.endsWith('/')) { + path = `${path.replace(/\/+$/, '')}/${encodeURIComponent(filename)}` + } else if (!path || path === '/') { + path = `/${encodeURIComponent(filename)}` + } + + const client = new FtpClient(30000) + try { + await client.access({ + host, + port, + user, + password, + secure + }) + await client.uploadFrom(stream, path) + return { httpStatus: 200, headers: {} } + } finally { + client.close() + } + } + + validate(): [boolean, string] { + const file = this.getFile() as FtpFileObject + if (!file.url) { + return [false, 'FTP URL is missing'] + } + try { + const parsed = new URL(file.url) + if (parsed.protocol !== 'ftp:' && parsed.protocol !== 'ftps:') { + return [false, 'URL must be ftp:// or ftps://'] + } + } catch { + return [false, 'Invalid FTP URL'] + } + if (this.config?.unsafeURLs) { + for (const regex of this.config.unsafeURLs) { + try { + // eslint-disable-next-line security/detect-non-literal-regexp + const pattern = new RegExp(regex) + if (pattern.test(file.url)) { + return [false, 'URL is marked as unsafe'] + } + } catch (e) { + /* ignore */ + } + } + } + return [true, ''] + } + + getDownloadUrl(): string { + if (this.validate()[0] === true) { + return this.getFile().url + } + return null + } + + async fetchSpecificFileMetadata( + fileObject: FtpFileObject, + _forceChecksum: boolean + ): Promise { + const { host, port, user, password, path, secure } = parseFtpUrl(fileObject.url) + const client = new FtpClient(30000) + try { + await client.access({ + host, + port, + user, + password, + secure + }) + let size = 0 + try { + size = await client.size(path) + } catch { + size = 0 + } + const name = path.split('/').filter(Boolean).pop() || '' + return { + valid: true, + contentLength: String(size >= 0 ? size : 0), + contentType: 'application/octet-stream', + name, + type: 'ftp', + encryptedBy: fileObject.encryptedBy, + encryptMethod: fileObject.encryptMethod + } + } finally { + client.close() + } + } +} diff --git a/src/components/storage/S3Storage.ts b/src/components/storage/S3Storage.ts index 2214d52b9..52e8e4e0d 100644 --- a/src/components/storage/S3Storage.ts +++ b/src/components/storage/S3Storage.ts @@ -5,6 +5,7 @@ import { } from '../../@types/fileObject.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' import { GetObjectCommand, HeadObjectCommand, S3Client } from '@aws-sdk/client-s3' +import { Upload } from '@aws-sdk/lib-storage' import { Readable } from 'stream' import { CORE_LOGGER } from '../../utils/logging/common.js' @@ -29,7 +30,7 @@ function createS3Client(s3Access: S3FileObject['s3Access']): S3Client { export class S3Storage extends Storage { public constructor(file: S3FileObject, config: OceanNodeConfig) { - super(file, config) + super(file, config, true) const [isValid, message] = this.validate() if (isValid === false) { throw new Error(`Error validating the S3 file: ${message}`) @@ -89,6 +90,37 @@ export class S3Storage extends Storage { } } + /** + * Upload a file via S3 multipart upload (streaming). If s3Access.objectKey ends with /, the key becomes objectKey + filename; otherwise objectKey is the target key. + * Uses @aws-sdk/lib-storage Upload so large streams are sent in parts without buffering the entire file. + */ + async upload( + filename: string, + stream: Readable + ): Promise<{ httpStatus: number; headers?: Record }> { + const { s3Access } = this.getFile() as S3FileObject + const s3Client = createS3Client(s3Access) + let key = s3Access.objectKey + if (key.endsWith('/')) { + key = `${key.replace(/\/+$/, '')}/${filename}` + } + const upload = new Upload({ + client: s3Client, + params: { + Bucket: s3Access.bucket, + Key: key, + Body: stream, + ContentType: 'application/octet-stream', + ContentDisposition: `attachment; filename="${filename.replace(/"/g, '\\"')}"` + }, + queueSize: 4, + partSize: 5 * 1024 * 1024, // 5MB minimum for S3 + leavePartsOnError: false + }) + await upload.done() + return { httpStatus: 200, headers: {} } + } + async fetchSpecificFileMetadata( fileObject: S3FileObject, _forceChecksum: boolean diff --git a/src/components/storage/Storage.ts b/src/components/storage/Storage.ts index baf0b513e..9d74dd44b 100644 --- a/src/components/storage/Storage.ts +++ b/src/components/storage/Storage.ts @@ -16,9 +16,15 @@ export abstract class Storage { private file: StorageObject config: OceanNodeConfig - public constructor(file: StorageObject, config: OceanNodeConfig) { + public hasUpload: boolean + public constructor( + file: StorageObject, + config: OceanNodeConfig, + hasUpload: boolean = false + ) { this.file = file this.config = config + this.hasUpload = hasUpload } abstract validate(): [boolean, string] diff --git a/src/components/storage/UrlStorage.ts b/src/components/storage/UrlStorage.ts index 2d0884eb9..509f24130 100644 --- a/src/components/storage/UrlStorage.ts +++ b/src/components/storage/UrlStorage.ts @@ -1,3 +1,4 @@ +import { Readable } from 'stream' import { FileInfoResponse, StorageReadable, @@ -11,7 +12,7 @@ import { Storage } from './Storage.js' export class UrlStorage extends Storage { public constructor(file: UrlFileObject, config: OceanNodeConfig) { - super(file, config) + super(file, config, true) const [isValid, message] = this.validate() if (isValid === false) { throw new Error(`Error validating the URL file: ${message}`) @@ -37,6 +38,40 @@ export class UrlStorage extends Storage { } } + /** + * Upload a file via HTTP PUT. Uses PUT regardless of UrlFileObject.method (which applies to download). + * @param filename – used in Content-Disposition and, if url ends with /, appended to url + * @param stream – readable stream to send as the request body + * @returns response status and headers + */ + async upload( + filename: string, + stream: Readable + ): Promise<{ httpStatus: number; headers?: Record }> { + const { url: baseUrl, headers: fileHeaders } = this.getFile() as UrlFileObject + let url = baseUrl + if (url.endsWith('/')) { + url = `${url.replace(/\/+$/, '')}/${encodeURIComponent(filename)}` + } + const headers: Record = { + ...(fileHeaders ?? {}), + 'Content-Disposition': `attachment; filename="${filename.replace(/"/g, '\\"')}"` + } + const response = await axios({ + method: 'put', + url, + data: stream, + headers, + timeout: 30000, + maxBodyLength: Infinity, + maxContentLength: Infinity + }) + return { + httpStatus: response.status, + headers: response.headers as Record + } + } + validate(): [boolean, string] { const file: UrlFileObject = this.getFile() as UrlFileObject if (!file.url || !file.method) { diff --git a/src/components/storage/getStorageClass.ts b/src/components/storage/getStorageClass.ts index e81120d31..fbad7f507 100644 --- a/src/components/storage/getStorageClass.ts +++ b/src/components/storage/getStorageClass.ts @@ -3,11 +3,17 @@ import { OceanNodeConfig } from '../../@types/OceanNode.js' import { CORE_LOGGER } from '../../utils/logging/common.js' import { ArweaveStorage } from './ArweaveStorage.js' +import { FTPStorage } from './FTPStorage.js' import { IpfsStorage } from './IpfsStorage.js' import { S3Storage } from './S3Storage.js' import { UrlStorage } from './UrlStorage.js' -export type StorageClass = UrlStorage | IpfsStorage | ArweaveStorage | S3Storage +export type StorageClass = + | UrlStorage + | IpfsStorage + | ArweaveStorage + | S3Storage + | FTPStorage export function getStorageClass(file: any, config: OceanNodeConfig): StorageClass { if (!file) { @@ -26,6 +32,8 @@ export function getStorageClass(file: any, config: OceanNodeConfig): StorageClas return new ArweaveStorage(file, config) case FileObjectType.S3: return new S3Storage(file, config) + case FileObjectType.FTP: + return new FTPStorage(file, config) default: throw new Error(`Invalid storage type: ${type}`) } diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts index b35b7bd56..62a1f310d 100644 --- a/src/components/storage/index.ts +++ b/src/components/storage/index.ts @@ -1,10 +1,11 @@ import { getStorageClass } from './getStorageClass.js' import { Storage } from './Storage.js' import { ArweaveStorage } from './ArweaveStorage.js' +import { FTPStorage } from './FTPStorage.js' import { IpfsStorage } from './IpfsStorage.js' import { S3Storage } from './S3Storage.js' import { UrlStorage } from './UrlStorage.js' Storage.getStorageClass = getStorageClass -export { Storage, UrlStorage, ArweaveStorage, IpfsStorage, S3Storage } +export { Storage, UrlStorage, ArweaveStorage, IpfsStorage, S3Storage, FTPStorage } diff --git a/src/test/data/assets.ts b/src/test/data/assets.ts index 532415d48..6f2175749 100644 --- a/src/test/data/assets.ts +++ b/src/test/data/assets.ts @@ -457,10 +457,10 @@ export const algoAsset = { version: '0.1', container: { entrypoint: 'node $ALGO', - image: 'node', - tag: 'latest', + image: 'ghcr.io/oceanprotocol/c2d_examples', + tag: 'js-general', checksum: - 'sha256:1155995dda741e93afe4b1c6ced2d01734a6ec69865cc0997daf1f4db7259a36' + 'sha256:75d2abe7651d54b074093e2cf44470d6c1abd7923eab08d86a0778f0a0ff9a6a' } } }, diff --git a/src/test/integration/algorithmsAccess.test.ts b/src/test/integration/algorithmsAccess.test.ts index 7af1a9b07..95fb54bd9 100644 --- a/src/test/integration/algorithmsAccess.test.ts +++ b/src/test/integration/algorithmsAccess.test.ts @@ -456,7 +456,7 @@ describe('Trusted algorithms Flow', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index e1c8a6561..98a1b8aa4 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -69,7 +69,7 @@ import ERC721Factory from '@oceanprotocol/contracts/artifacts/contracts/ERC721Fa import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC721Template.sol/ERC721Template.json' with { type: 'json' } import OceanToken from '@oceanprotocol/contracts/artifacts/contracts/utils/OceanToken.sol/OceanToken.json' with { type: 'json' } import EscrowJson from '@oceanprotocol/contracts/artifacts/contracts/escrow/Escrow.sol/Escrow.json' with { type: 'json' } -import { createHash } from 'crypto' +import { createHash, randomBytes } from 'crypto' import { EncryptMethod } from '../../@types/fileObject.js' import { getAlgoChecksums, @@ -84,6 +84,43 @@ import { createHashForSignature, safeSign } from '../utils/signature.js' const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) +/** + * Polls getComputeEnvironments until every environment's resources (and free.resources) + * have inUse === 0. Use with the same pattern as the compute tests: pass a callback that + * calls ComputeGetEnvironmentsHandler and streamToObject. + */ +export async function waitForAllJobsToFinish( + oceanNode: OceanNode, + options?: { pollIntervalMs?: number; timeoutMs?: number } +): Promise { + const getEnvironmentsTask = { + command: PROTOCOL_COMMANDS.COMPUTE_GET_ENVIRONMENTS + } + const pollIntervalMs = options?.pollIntervalMs ?? 2000 + const timeoutMs = options?.timeoutMs ?? 120_000 + const deadline = Date.now() + timeoutMs + + while (true) { + const response = await new ComputeGetEnvironmentsHandler(oceanNode).handle( + getEnvironmentsTask + ) + const envs = await streamToObject(response.stream as Readable) + + const allIdle = envs.every((env: ComputeEnvironment) => { + const resources = env.resources ?? [] + const freeResources = env.free?.resources ?? [] + const paidInUse = resources.every((r) => (r.inUse ?? 0) === 0) + const freeInUse = freeResources.every((r) => (r.inUse ?? 0) === 0) + return paidInUse && freeInUse + }) + if (allIdle) return + if (Date.now() >= deadline) { + throw new Error(`waitForAllJobsToFinish timed out after ${timeoutMs}ms`) + } + await sleep(pollIntervalMs) + } +} + describe('Compute', () => { let previousConfiguration: OverrideEnvConfig[] let config: OceanNodeConfig @@ -99,6 +136,7 @@ describe('Compute', () => { let publishedAlgoDataset: any let jobId: string let freeJobId: string + let jobWithOutputURL: string let datasetOrderTxId: any let algoOrderTxId: any let paymentToken: any @@ -637,6 +675,112 @@ describe('Compute', () => { assert(response.status.httpStatus === 500, 'Failed to get 500 response') assert(!response.stream, 'We should not have a stream') }) + it('should start a compute job with output to URL storage at 172.15.0.7', async () => { + // deposit funds and create auth in escrow + let balance = await paymentTokenContract.balanceOf(await consumerAccount.getAddress()) + if (BigInt(balance.toString()) === BigInt(0)) { + const mintAmount = ethers.parseUnits('1000', 18) + const mintTx = await paymentTokenContract.mint( + await consumerAccount.getAddress(), + mintAmount + ) + await mintTx.wait() + balance = await paymentTokenContract.balanceOf(await consumerAccount.getAddress()) + } + await paymentTokenContract + .connect(consumerAccount) + .approve(initializeResponse.payment.escrowAddress, balance) + await escrowContract + .connect(consumerAccount) + .deposit(initializeResponse.payment.token, balance) + + await escrowContract + .connect(consumerAccount) + .authorize( + initializeResponse.payment.token, + firstEnv.consumerAddress, + balance, + initializeResponse.payment.minLockSeconds, + 10 + ) + + const fundsBefore = await oceanNode.escrow.getUserAvailableFunds( + DEVELOPMENT_CHAIN_ID, + await consumerAccount.getAddress(), + paymentToken + ) + assert(BigInt(fundsBefore.toString()) > BigInt(0), 'Should have funds in escrow') + + const computeOutput = { + remoteStorage: { + type: 'url', + url: 'http://172.15.0.7:80/', + method: 'get' + }, + encryption: { + encryptMethod: EncryptMethod.AES, + key: randomBytes(32).toString('hex') + } + } + const encryptedOutput = await oceanNode + .getKeyManager() + .encrypt( + new Uint8Array(Buffer.from(JSON.stringify(computeOutput))), + EncryptMethod.ECIES + ) + + const nonce = Date.now().toString() + const messageHashBytes = createHashForSignature( + await consumerAccount.getAddress(), + nonce, + PROTOCOL_COMMANDS.COMPUTE_START + ) + const signature = await safeSign(consumerAccount, messageHashBytes) + const re = [] + for (const res of firstEnv.resources) { + re.push({ id: res.id, amount: res.min }) + } + const startComputeTask: PaidComputeStartCommand = { + command: PROTOCOL_COMMANDS.COMPUTE_START, + consumerAddress: await consumerAccount.getAddress(), + signature, + nonce, + environment: firstEnv.id, + datasets: [ + { + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: datasetOrderTxId + } + ], + algorithm: { + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: algoOrderTxId, + meta: publishedAlgoDataset.ddo.metadata.algorithm + }, + output: Buffer.from(encryptedOutput).toString('hex'), + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + metadata: { key: 'value' }, + additionalViewers: [await additionalViewerAccount.getAddress()], + maxJobDuration: computeJobDuration, + resources: re + } + const response = await new PaidComputeStartHandler(oceanNode).handle(startComputeTask) + assert(response, 'Failed to get response') + assert( + response.status.httpStatus === 200, + `Expected 200, got ${response.status.httpStatus}: ${response.status?.error ?? ''}` + ) + assert(response.stream, 'Failed to get stream') + expect(response.stream).to.be.instanceOf(Readable) + const jobs = await streamToObject(response.stream as Readable) + assert(jobs[0].jobId, 'Failed to get job id') + jobWithOutputURL = jobs[0].jobId + }) it('should fail to start a compute job without escrow funds', async () => { // ensure clean escrow state: no funds, no auths, no locks @@ -705,7 +849,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken @@ -722,17 +866,27 @@ describe('Compute', () => { assert(!response.stream, 'We should not have a stream') }) - it('should start a compute job with maxed resources', async () => { - // deposit funds and create auth in escrow - const balance = await paymentTokenContract.balanceOf( - await consumerAccount.getAddress() - ) + it('should start a compute job with maxed resources', async function () { + this.timeout(130_000) // waitForAllJobsToFinish can take up to 120s + await waitForAllJobsToFinish(oceanNode) + let balance = await paymentTokenContract.balanceOf(await consumerAccount.getAddress()) + if (BigInt(balance.toString()) === BigInt(0)) { + console.log('Minting') + const mintAmount = ethers.parseUnits('1000', 18) + const mintTx = await paymentTokenContract.mint( + await consumerAccount.getAddress(), + mintAmount + ) + await mintTx.wait() + balance = await paymentTokenContract.balanceOf(await consumerAccount.getAddress()) + } await paymentTokenContract .connect(consumerAccount) .approve(initializeResponse.payment.escrowAddress, balance) await escrowContract .connect(consumerAccount) .deposit(initializeResponse.payment.token, balance) + await escrowContract .connect(consumerAccount) .authorize( @@ -742,7 +896,6 @@ describe('Compute', () => { initializeResponse.payment.minLockSeconds, 10 ) - const auth = await oceanNode.escrow.getAuthorizations( DEVELOPMENT_CHAIN_ID, paymentToken, @@ -806,7 +959,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken @@ -901,7 +1054,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken @@ -947,7 +1100,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, queueMaxWaitTime: 300 // 5 minutes // additionalDatasets?: ComputeAsset[] // output?: ComputeOutput @@ -1742,7 +1895,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, encryptedDockerRegistryAuth: encryptedAuth } @@ -1799,7 +1952,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, encryptedDockerRegistryAuth: encryptedAuth } @@ -2013,6 +2166,53 @@ describe('Compute', () => { }) }) + it('should wait for jobWithOutputURL status 70 and download output from URL', async function () { + this.timeout(130_000) // waitForAllJobsToFinish can take up to 120s + assert(jobWithOutputURL, 'jobWithOutputURL must be set by previous test') + const statusTask: ComputeGetStatusCommand = { + command: PROTOCOL_COMMANDS.COMPUTE_GET_STATUS, + consumerAddress: null, + agreementId: null, + jobId: jobWithOutputURL + } + const deadline = Date.now() + DEFAULT_TEST_TIMEOUT + let status: number | null = null + while (Date.now() < deadline) { + const response = await new ComputeGetStatusHandler(oceanNode).handle(statusTask) + assert(response?.status?.httpStatus === 200, 'Failed to get status') + const { stream } = response + const jobs = await streamToObject(stream as Readable) + const [job] = jobs + if (job) { + const { status: jobStatus } = job + if (jobStatus !== undefined) { + status = jobStatus + if ( + status === C2DStatusNumber.JobFinished || + status === C2DStatusNumber.JobSettle + ) + break + } + } + await new Promise((resolve) => setTimeout(resolve, 3000)) + } + assert( + status === C2DStatusNumber.JobFinished || status === C2DStatusNumber.JobSettle, + `Job ${jobWithOutputURL} did not reach status 70 (JobFinished) in time (last status: ${status})` + ) + const outputUrl = `http://172.15.0.7:80/outputs-${jobWithOutputURL}.tar` + const downloadResponse = await fetch(outputUrl) + assert( + downloadResponse.ok, + `Failed to download output from ${outputUrl}: ${downloadResponse.status} ${downloadResponse.statusText}` + ) + const body = await downloadResponse.arrayBuffer() + assert(body.byteLength > 0, `Output file at ${outputUrl} should be non-empty`) + console.log( + `**** Downloaded output from ${outputUrl}, size: ${body.byteLength} bytes` + ) + }) + after(async () => { await tearDownEnvironment(previousConfiguration) indexer.stopAllChainIndexers() @@ -2107,7 +2307,7 @@ describe('Compute Access Restrictions', () => { serviceId: publishedAlgoDataset.ddo.services[0].id, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {} + output: null } } @@ -2395,7 +2595,6 @@ describe('Compute Access Restrictions', () => { firstEnv.id ) const response = await new PaidComputeStartHandler(oceanNode).handle(command) - console.log(response) expect(response.status.httpStatus).to.not.equal(403) }) @@ -2406,7 +2605,6 @@ describe('Compute Access Restrictions', () => { firstEnv.id ) const response = await new PaidComputeStartHandler(oceanNode).handle(command) - console.log(response) assert( response.status.httpStatus === 403, `Expected 403 but got ${response.status.httpStatus}: ${response.status.error}` diff --git a/src/test/integration/storage/arweaveStorage.test.ts b/src/test/integration/storage/arweaveStorage.test.ts new file mode 100644 index 000000000..897d31c33 --- /dev/null +++ b/src/test/integration/storage/arweaveStorage.test.ts @@ -0,0 +1,148 @@ +/** + * ArweaveStorage integration tests. + * Moved from unit/storage.test.ts. + */ + +import { Storage, ArweaveStorage } from '../../../components/storage/index.js' +import { FileInfoRequest, FileObjectType } from '../../../@types/fileObject.js' +import { expect, assert } from 'chai' +import { + OverrideEnvConfig, + buildEnvOverrideConfig, + tearDownEnvironment, + DEFAULT_TEST_TIMEOUT +} from '../../utils/utils.js' +import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js' +import { getConfiguration } from '../../../utils/index.js' + +describe('Arweave Storage integration tests', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let file: any = { + type: 'arweave', + transactionId: '0x2563ed54abc0001bcaef' + } + let error: Error + let previousConfiguration: OverrideEnvConfig[] + let config: Awaited> + + before(async () => { + previousConfiguration = buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.ARWEAVE_GATEWAY], + ['https://snaznabndfe3.arweave.net/nnLNdp6nuTb8mJ-qOgbUEx-9SBtBXQc_jejYOWzYEkM'] + ) + config = await getConfiguration() + }) + + it('Storage instance', () => { + expect(Storage.getStorageClass(file, config)).to.be.instanceOf(ArweaveStorage) + }) + + it('Arweave validation passes', () => { + expect(Storage.getStorageClass(file, config).validate()).to.eql([true, '']) + }) + + it('Arweave validation fails', () => { + file = { + type: 'arweave' + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql( + 'Error validating the Arweave file: Missing transaction ID' + ) + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) + +describe('Arweave Storage getFileInfo integration tests', function () { + let storage: ArweaveStorage + + before(async () => { + const config = await getConfiguration(true) + storage = new ArweaveStorage( + { + type: FileObjectType.ARWEAVE, + transactionId: 'gPPDyusRh2ZyFl-sQ2ODK6hAwCRBAOwp0OFKr0n23QE' + }, + config + ) + }) + + it('Successfully retrieves file info for an Arweave transaction', async () => { + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.ARWEAVE + } + const fileInfo = await storage.getFileInfo(fileInfoRequest) + + assert(fileInfo[0].valid, 'File info is valid') + assert(fileInfo[0].type === FileObjectType.ARWEAVE, 'Type is incorrect') + assert( + fileInfo[0].contentType === 'text/csv; charset=utf-8' || + fileInfo[0].contentType === 'text/csv', + 'Content type is incorrect' + ) + assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect') + }) + + it('Throws error when transaction ID is missing in request', async () => { + const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE } + try { + await storage.getFileInfo(fileInfoRequest) + } catch (err) { + expect(err.message).to.equal('Transaction ID is required for type arweave') + } + }) +}) + +describe('Arweave Storage with malformed transaction ID integration tests', () => { + let error: Error + let config: Awaited> + + before(async () => { + config = await getConfiguration() + }) + + it('should detect URL path format', () => { + try { + // eslint-disable-next-line no-new + new ArweaveStorage( + { + type: 'arweave', + transactionId: + 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the Arweave file: Transaction ID looks like an URL. Please specify URL storage instead.' + ) + }) + + it('should detect path regex', () => { + try { + // eslint-disable-next-line no-new + new ArweaveStorage( + { + type: 'arweave', + transactionId: '../../myFolder/' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the Arweave file: Transaction ID looks like a file path' + ) + }) +}) diff --git a/src/test/integration/storage/ftpStorage.test.ts b/src/test/integration/storage/ftpStorage.test.ts new file mode 100644 index 000000000..a6d2c2b0a --- /dev/null +++ b/src/test/integration/storage/ftpStorage.test.ts @@ -0,0 +1,121 @@ +/** + * FTPStorage integration tests. + * + * Uses vsftpd at 172.15.0.7 (ports 20/21) with user ftpuser / ftppass. before() uploads + * readme.txt first so getReadableStream and getFileInfo have a file to read. + */ + +import { Readable } from 'stream' +import { Storage, FTPStorage } from '../../../components/storage/index.js' +import { FileInfoRequest, FileObjectType } from '../../../@types/fileObject.js' +import { expect, assert } from 'chai' +import { getConfiguration } from '../../../utils/index.js' +import { DEFAULT_TEST_TIMEOUT } from '../../utils/utils.js' + +const FTP_HOST = '172.15.0.7' +const FTP_PORT = 21 +const FTP_USER = 'ftpuser' +const FTP_PASS = 'ftppass' +const FTP_BASE_URL = `ftp://${FTP_USER}:${FTP_PASS}@${FTP_HOST}:${FTP_PORT}` +const FTP_FILE_URL = `${FTP_BASE_URL}/readme.txt` +const FTP_UPLOAD_DIR = `${FTP_BASE_URL}` + +describe('FTP Storage integration tests', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let config: Awaited> + let error: Error + + before(async function () { + config = await getConfiguration() + const storage = new FTPStorage({ type: 'ftp', url: FTP_UPLOAD_DIR }, config) + await storage.upload('readme.txt', Readable.from(['FTP test file content'])) + }) + + it('returns FTPStorage from getStorageClass for type ftp', () => { + const file = { + type: 'ftp', + url: 'ftp://example.com/path/to/file.txt' + } + const storage = Storage.getStorageClass(file, config) + expect(storage).to.be.instanceOf(FTPStorage) + }) + + it('FTP validation passes for valid ftp URL', () => { + const file = { + type: 'ftp', + url: 'ftp://user:pass@ftp.example.com:21/files/data.zip' + } + const storage = Storage.getStorageClass(file, config) as FTPStorage + expect(storage.validate()).to.eql([true, '']) + }) + + it('FTP validation passes for valid ftps URL', () => { + const file = { + type: 'ftp', + url: 'ftps://secure.example.com/pub/readme.txt' + } + const storage = Storage.getStorageClass(file, config) as FTPStorage + expect(storage.validate()).to.eql([true, '']) + }) + + it('FTP validation fails when URL is missing', () => { + const file = { type: 'ftp' } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.include('FTP URL is missing') + }) + + it('FTP validation fails for non-ftp URL', () => { + const file = { + type: 'ftp', + url: 'https://example.com/file.txt' + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.include('URL must be ftp:// or ftps://') + }) + + it('getDownloadUrl returns the FTP URL', () => { + const file = { + type: 'ftp', + url: 'ftp://host.example.com/dir/file.bin' + } + const storage = Storage.getStorageClass(file, config) as FTPStorage + expect(storage.getDownloadUrl()).to.equal('ftp://host.example.com/dir/file.bin') + }) + + it('getReadableStream connects to vsftpd and returns stream', async function () { + const file = { type: 'ftp', url: FTP_FILE_URL } + const storage = Storage.getStorageClass(file, config) as FTPStorage + const result = await storage.getReadableStream() + expect(result).to.have.property('stream') + expect(result.httpStatus).to.equal(200) + }) + + it('getFileInfo returns metadata from vsftpd', async function () { + const file = { type: 'ftp', url: FTP_FILE_URL } + const storage = Storage.getStorageClass(file, config) as FTPStorage + const fileInfoRequest: FileInfoRequest = { type: FileObjectType.FTP } + const fileInfo = await storage.getFileInfo(fileInfoRequest) + expect(fileInfo).to.have.lengthOf(1) + assert(fileInfo[0].valid, 'File info is valid') + expect(fileInfo[0].type).to.equal('ftp') + expect(fileInfo[0].contentType).to.equal('application/octet-stream') + }) + + it('upload sends file via FTP to vsftpd', async function () { + const storage = new FTPStorage({ type: 'ftp', url: FTP_UPLOAD_DIR }, config) + const filename = `ftp-upload-test-${Date.now()}.txt` + const stream = Readable.from(['FTP upload test content']) + const result = await storage.upload(filename, stream) + expect(result).to.have.property('httpStatus') + expect(result.httpStatus).to.equal(200) + }) +}) diff --git a/src/test/integration/storage/ipfsStorage.test.ts b/src/test/integration/storage/ipfsStorage.test.ts new file mode 100644 index 000000000..b440df0b0 --- /dev/null +++ b/src/test/integration/storage/ipfsStorage.test.ts @@ -0,0 +1,237 @@ +/** + * IpfsStorage integration tests. + * Moved from unit/storage.test.ts. + */ + +import { Storage, IpfsStorage } from '../../../components/storage/index.js' +import { + FileInfoRequest, + FileObjectType, + EncryptMethod +} from '../../../@types/fileObject.js' +import { expect, assert } from 'chai' +import { + OverrideEnvConfig, + buildEnvOverrideConfig, + tearDownEnvironment, + setupEnvironment, + DEFAULT_TEST_TIMEOUT +} from '../../utils/utils.js' +import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js' +import { getConfiguration } from '../../../utils/index.js' +import { expectedTimeoutFailure } from '../testUtils.js' + +const nodeId = '16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72' +const nodeId2 = '16Uiu2HAmQWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq73' + +describe('IPFS Storage integration tests', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let file: any = { + type: 'ipfs', + hash: 'Qxchjkflsejdfklgjhfkgjkdjoiderj' + } + let error: Error + let previousConfiguration: OverrideEnvConfig[] + let config: Awaited> + + before(async () => { + previousConfiguration = buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], + ['https://ipfs.oceanprotocol.com'] + ) + config = await getConfiguration() + }) + + it('Storage instance', () => { + expect(Storage.getStorageClass(file, config)).to.be.instanceOf(IpfsStorage) + }) + + it('IPFS validation passes', () => { + expect(Storage.getStorageClass(file, config).validate()).to.eql([true, '']) + }) + + it('IPFS validation fails', () => { + file = { + type: 'ipfs' + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql('Error validating the IPFS file: Missing CID') + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) + +describe('IPFS Storage with malformed hash integration tests', () => { + let error: Error + let config: Awaited> + + before(async () => { + config = await getConfiguration() + }) + + it('should detect URL path format', () => { + try { + // eslint-disable-next-line no-new + new IpfsStorage( + { + type: 'ipfs', + hash: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the IPFS file: CID looks like an URL. Please specify URL storage instead.' + ) + }) + + it('should detect path regex', () => { + try { + // eslint-disable-next-line no-new + new IpfsStorage( + { + type: 'ipfs', + hash: '../../myFolder/' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the IPFS file: CID looks like a file path' + ) + }) +}) + +describe('IPFS Storage getFileInfo integration tests', function () { + let storage: IpfsStorage + let previousConfiguration: OverrideEnvConfig[] + let config: Awaited> + + before(async () => { + previousConfiguration = await buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], + ['https://ipfs.oceanprotocol.com'] + ) + await setupEnvironment(undefined, previousConfiguration) + config = await getConfiguration() + storage = new IpfsStorage( + { + type: FileObjectType.IPFS, + hash: 'QmRhsp7eghZtW4PktPC2wAHdKoy2LiF1n6UXMKmAhqQJUA' + }, + config + ) + }) + + it('Successfully retrieves file info for an IPFS hash', function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 2) + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.IPFS + } + setTimeout(async () => { + const fileInfo = await storage.getFileInfo(fileInfoRequest) + if (fileInfo && fileInfo.length > 0) { + assert(fileInfo[0].valid, 'File info is valid') + assert(fileInfo[0].type === 'ipfs', 'Type is incorrect') + if (fileInfo[0].contentType && fileInfo[0].contentLength) { + assert(fileInfo[0].contentType === 'text/csv', 'Content type is incorrect') + assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect') + } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(true) + } + }, DEFAULT_TEST_TIMEOUT) + }) + + it('Throws error when hash is missing in request', async () => { + const fileInfoRequest: FileInfoRequest = { type: FileObjectType.IPFS } + try { + await storage.getFileInfo(fileInfoRequest) + } catch (err) { + expect(err.message).to.equal('Hash is required for type ipfs') + } + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) + +describe('IPFS Storage encryption integration tests', function () { + this.timeout(15000) + + let storage: IpfsStorage + let previousConfiguration: OverrideEnvConfig[] + + before(async () => { + previousConfiguration = await buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], + ['https://ipfs.oceanprotocol.com'] + ) + await setupEnvironment(undefined, previousConfiguration) + const config = await getConfiguration() + storage = new IpfsStorage( + { + type: 'ipfs', + hash: 'QmQVPuoXMbVEk7HQBth5pGPPMcgvuq4VSgu2XQmzU5M2Pv', + encryptedBy: nodeId, + encryptMethod: EncryptMethod.AES + }, + config + ) + }) + + it('isEncrypted should return true for an encrypted file', () => { + assert(storage.isEncrypted() === true, 'invalid response to isEncrypted()') + }) + + it('canDecrypt should return true for this node', () => { + assert( + storage.canDecrypt(nodeId) === true, + 'Wrong response from canDecrypt() for an encrypted file' + ) + }) + + it('File info includes encryptedBy and encryptMethod', function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 2) + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.IPFS + } + setTimeout(async () => { + const fileInfo = await storage.getFileInfo(fileInfoRequest) + if (fileInfo && fileInfo.length > 0) { + assert(fileInfo[0].valid, 'File info is valid') + expect(fileInfo[0].type).to.equal('ipfs') + if (fileInfo[0].contentType && fileInfo[0].encryptedBy) { + expect(fileInfo[0].contentType).to.equal('application/octet-stream') + expect(fileInfo[0].encryptedBy).to.equal(nodeId) + expect(fileInfo[0].encryptMethod).to.equal(EncryptMethod.AES) + } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(true) + } + }, DEFAULT_TEST_TIMEOUT) + }) + + it('canDecrypt should return false when called from an unauthorised node', () => { + assert( + storage.canDecrypt(nodeId) === true, + 'Wrong response from canDecrypt() for an unencrypted file' + ) + assert( + storage.canDecrypt(nodeId2) === false, + 'Wrong response from canDecrypt() for an unencrypted file' + ) + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) diff --git a/src/test/integration/storage/s3Storage.test.ts b/src/test/integration/storage/s3Storage.test.ts new file mode 100644 index 000000000..76f2640db --- /dev/null +++ b/src/test/integration/storage/s3Storage.test.ts @@ -0,0 +1,286 @@ +/** + * S3 Storage integration tests against Ceph RGW (or any S3-compatible endpoint). + * + * Prerequisites: + * - Ceph RGW (or compatible) running, e.g. at http://172.15.0.7:7480 + * - Environment variables set: + * - S3_TEST_ENDPOINT (optional, default: http://172.15.0.7:7480) + * - S3_TEST_ACCESS_KEY_ID + * - S3_TEST_SECRET_ACCESS_KEY + * - S3_TEST_BUCKET + * + * The test creates the bucket if missing, puts a temporary object, then removes the object in after(). + * If credentials are missing or setup fails (endpoint unreachable, auth failure), the suite is skipped. + */ + +import { expect } from 'chai' +import { Readable } from 'stream' +import { Storage, S3Storage } from '../../../components/storage/index.js' +import { getConfiguration } from '../../../utils/index.js' +import { + CreateBucketCommand, + DeleteObjectCommand, + PutObjectCommand, + S3Client +} from '@aws-sdk/client-s3' +import { FileInfoRequest, FileObjectType } from '../../../@types/fileObject.js' +import { DEFAULT_TEST_TIMEOUT } from '../../utils/utils.js' + +const S3_TEST_ENDPOINT = 'http://172.15.0.7:7480' +const S3_TEST_ACCESS_KEY_ID = 'ocean123' +const S3_TEST_SECRET_ACCESS_KEY = 'ocean123secret' +const S3_TEST_BUCKET = 'testbucket' +const TEST_OBJECT_KEY = 'integration-test/hello.txt' +const TEST_BODY = 'Hello S3 from integration test' + +function canRunS3Tests(): boolean { + return Boolean(S3_TEST_ACCESS_KEY_ID && S3_TEST_SECRET_ACCESS_KEY && S3_TEST_BUCKET) +} + +function createTestS3Client(): S3Client { + return new S3Client({ + endpoint: S3_TEST_ENDPOINT, + region: 'us-east-1', + forcePathStyle: true, + credentials: { + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY! + } + }) +} + +describe('S3 Storage integration (Ceph RGW)', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let config: Awaited> + let s3Client: S3Client + let objectCreated = false + const uploadTestKeys: string[] = [] + + before(async function () { + if (!canRunS3Tests()) { + this.skip() + return + } + config = await getConfiguration() + s3Client = createTestS3Client() + try { + await s3Client.send(new CreateBucketCommand({ Bucket: S3_TEST_BUCKET! })) + } catch (err: any) { + const alreadyExists = + err.name === 'BucketAlreadyExists' || + err.Code === 'BucketAlreadyExists' || + err.Code === 'BucketAlreadyOwnedByYou' || + err.$metadata?.httpStatusCode === 409 + if (!alreadyExists) { + this.skip() + return + } + } + try { + await s3Client.send( + new PutObjectCommand({ + Bucket: S3_TEST_BUCKET, + Key: TEST_OBJECT_KEY, + Body: TEST_BODY, + ContentType: 'text/plain' + }) + ) + objectCreated = true + } catch (err: any) { + this.skip() + } + }) + + after(async function () { + if (!s3Client) return + try { + if (objectCreated) { + await s3Client.send( + new DeleteObjectCommand({ + Bucket: S3_TEST_BUCKET, + Key: TEST_OBJECT_KEY + }) + ) + } + for (const key of uploadTestKeys) { + await s3Client.send(new DeleteObjectCommand({ Bucket: S3_TEST_BUCKET, Key: key })) + } + } catch { + // ignore cleanup errors + } + }) + + it('returns S3Storage from getStorageClass for type s3', function () { + if (!canRunS3Tests()) this.skip() + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: TEST_OBJECT_KEY, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) + expect(storage).to.be.instanceOf(S3Storage) + }) + + it('validates S3 file object', function () { + if (!canRunS3Tests()) this.skip() + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: TEST_OBJECT_KEY, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + expect(storage.validate()).to.eql([true, '']) + }) + + it('gets readable stream and reads body', async function () { + if (!canRunS3Tests()) this.skip() + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: TEST_OBJECT_KEY, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + const result = await storage.getReadableStream() + expect(result.httpStatus).to.equal(200) + expect(result.stream).to.be.instanceOf(Readable) + const chunks: Buffer[] = [] + for await (const chunk of result.stream as Readable) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + } + const body = Buffer.concat(chunks).toString('utf8') + expect(body).to.equal(TEST_BODY) + }) + + it('fetches file metadata via fetchSpecificFileMetadata', async function () { + if (!canRunS3Tests()) this.skip() + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: TEST_OBJECT_KEY, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + const meta = await storage.fetchSpecificFileMetadata(file, false) + expect(meta.valid).to.equal(true) + expect(meta.type).to.equal('s3') + expect(meta.contentLength).to.equal(String(TEST_BODY.length)) + expect(meta.contentType).to.equal('text/plain') + expect(meta.name).to.equal('hello.txt') + }) + + it('getFileInfo returns file info for S3', async function () { + if (!canRunS3Tests()) this.skip() + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: TEST_OBJECT_KEY, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + const fileInfoRequest: FileInfoRequest = { type: FileObjectType.S3 } + const fileInfo = await storage.getFileInfo(fileInfoRequest) + expect(fileInfo).to.have.lengthOf(1) + expect(fileInfo[0].valid).to.equal(true) + expect(fileInfo[0].contentLength).to.equal(String(TEST_BODY.length)) + expect(fileInfo[0].contentType).to.equal('text/plain') + }) + + it('upload sends file via PutObject and returns status', async function () { + if (!canRunS3Tests()) this.skip() + const key = `integration-test/upload-${Date.now()}.txt` + uploadTestKeys.push(key) + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: key, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + const body = 'S3 upload test content' + const result = await storage.upload('upload.txt', Readable.from([body])) + expect(result).to.have.property('httpStatus') + expect(result.httpStatus).to.equal(200) + }) + + it('upload with objectKey ending in / appends filename', async function () { + if (!canRunS3Tests()) this.skip() + const prefix = 'integration-test/upload-dir/' + const filename = 'appended.txt' + uploadTestKeys.push(`${prefix}${filename}`) + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: prefix, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + const result = await storage.upload(filename, Readable.from(['data'])) + expect(result.httpStatus).to.equal(200) + }) + + it('upload then getReadableStream returns uploaded content', async function () { + if (!canRunS3Tests()) this.skip() + const key = `integration-test/roundtrip-${Date.now()}.txt` + uploadTestKeys.push(key) + const uploadBody = 'Roundtrip test body' + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: key, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + await storage.upload('roundtrip.txt', Readable.from([uploadBody])) + const result = await storage.getReadableStream() + expect(result.httpStatus).to.equal(200) + const chunks: Buffer[] = [] + for await (const chunk of result.stream as Readable) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + } + expect(Buffer.concat(chunks).toString('utf8')).to.equal(uploadBody) + }) +}) diff --git a/src/test/integration/storage/urlStorage.test.ts b/src/test/integration/storage/urlStorage.test.ts new file mode 100644 index 000000000..e46f8857e --- /dev/null +++ b/src/test/integration/storage/urlStorage.test.ts @@ -0,0 +1,377 @@ +/** + * UrlStorage integration tests. + * + * Includes tests moved from unit/storage.test.ts and upload tests against + * an Apache server. Upload tests use http://172.15.0.7:80 (Apache must allow PUT). + * If the server is unreachable or rejects PUT, upload tests are skipped. + */ + +import { Readable } from 'stream' +import { Storage, UrlStorage } from '../../../components/storage/index.js' +import { + FileInfoRequest, + FileObjectType, + EncryptMethod +} from '../../../@types/fileObject.js' +import { expect, assert } from 'chai' +import { + OverrideEnvConfig, + buildEnvOverrideConfig, + tearDownEnvironment, + setupEnvironment, + DEFAULT_TEST_TIMEOUT +} from '../../utils/utils.js' +import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js' +import { getConfiguration } from '../../../utils/index.js' + +const nodeId = '16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72' +const APACHE_BASE_URL = 'http://172.15.0.7:80' + +describe('URL Storage integration tests', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let file: any = { + type: 'url', + url: 'http://someUrl.com/file.json', + method: 'get', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + }, + encryptedBy: nodeId, + encryptMethod: EncryptMethod.AES + } + let storage: UrlStorage + let error: Error + let config: Awaited> + + before(async () => { + config = await getConfiguration() + storage = Storage.getStorageClass(file, config) as UrlStorage + }) + + it('Storage instance', () => { + expect(storage).to.be.instanceOf(UrlStorage) + }) + + it('URL validation passes', () => { + expect(storage.validate()).to.eql([true, '']) + }) + + it('isEncrypted should return true for an encrypted file', () => { + assert(storage.isEncrypted() === true, 'invalid response to isEncrypted()') + }) + + it('canDecrypt should return true for the correct nodeId', () => { + assert(storage.canDecrypt(nodeId) === true, "can't decrypt with the correct nodeId") + }) + + it('canDecrypt should return false for an incorrect nodeId', () => { + assert( + storage.canDecrypt('wrongNodeId') === false, + 'can decrypt with the wrong nodeId' + ) + }) + + it('URL validation fails on missing URL', () => { + file = { + type: 'url', + method: 'get', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql( + 'Error validating the URL file: URL or method are missing' + ) + file = { + type: 'url', + url: 'http://someUrl.com/file.json', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql( + 'Error validating the URL file: URL or method are missing' + ) + }) + + it('URL validation fails on invalid method', () => { + file = { + type: 'url', + url: 'http://someUrl.com/file.json', + method: 'put', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql('Error validating the URL file: Invalid method for URL') + }) + + it('URL validation fails on filename', () => { + file = { + type: 'url', + url: './../dir/file.json', + method: 'get', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql( + 'Error validating the URL file: URL looks like a file path' + ) + }) + + it('Gets download URL', () => { + file = { + type: 'url', + url: 'http://someUrl.com/file.json', + method: 'get', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + storage = Storage.getStorageClass(file, config) as UrlStorage + expect(storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json') + }) + + it('Gets readable stream', async () => { + file = { + type: 'url', + url: 'https://stock-api.oceanprotocol.com/stock/stock.json', + method: 'get' + } + const storageInstance = Storage.getStorageClass(file, config) + const stream = await storageInstance.getReadableStream() + expect(stream).not.to.eql(null) + }) + + it('Gets readable stream with headers as plain object', async () => { + file = { + type: 'url', + url: 'https://stock-api.oceanprotocol.com/stock/stock.json', + method: 'get', + headers: { 'X-Test-Header': 'test' } + } + const storageInstance = Storage.getStorageClass(file, config) + const stream = await storageInstance.getReadableStream() + expect(stream).not.to.eql(null) + }) +}) + +describe('Unsafe URL integration tests', () => { + let previousConfiguration: OverrideEnvConfig[] + let file: any + let error: Error + let config: Awaited> + + before(async () => { + previousConfiguration = await setupEnvironment( + null, + buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.UNSAFE_URLS], + [JSON.stringify(['^.*(169.254.169.254).*', '^.*(127.0.0.1).*'])] + ) + ) + config = await getConfiguration(true) + }) + + it('Should reject unsafe URL', () => { + file = { + type: 'url', + url: 'http://169.254.169.254/asfd', + method: 'get' + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql('Error validating the URL file: URL is marked as unsafe') + }) + + it('Should allow safe URL', () => { + file = { + type: 'url', + url: 'https://oceanprotocol.com', + method: 'get' + } + const storageInstance = Storage.getStorageClass(file, config) as UrlStorage + expect(storageInstance.getDownloadUrl()).to.eql('https://oceanprotocol.com') + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) + +describe('URL Storage getFileInfo integration tests', () => { + let storage: UrlStorage + + before(async () => { + const config = await getConfiguration() + storage = new UrlStorage( + { + type: 'url', + url: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', + method: 'get' + }, + config + ) + }) + + it('isEncrypted should return false for an unencrypted file', () => { + assert(storage.isEncrypted() === false, 'invalid response to isEncrypted()') + }) + + it('canDecrypt should return false when the file is not encrypted', () => { + assert( + storage.canDecrypt('16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72') === + false, + 'Wrong response from canDecrypt() for an unencrypted file' + ) + }) + + it('Successfully retrieves file info for a URL', async () => { + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.URL + } + const fileInfo = await storage.getFileInfo(fileInfoRequest) + + assert(fileInfo[0].valid, 'File info is valid') + expect(fileInfo[0].contentLength).to.equal('319520') + expect(fileInfo[0].contentType).to.equal('text/plain; charset=utf-8') + expect(fileInfo[0].name).to.equal('shs_dataset_test.txt') + expect(fileInfo[0].type).to.equal('url') + }) +}) + +describe('URL Storage with malformed URL integration tests', () => { + let error: Error + let config: Awaited> + + before(async () => { + config = await getConfiguration() + }) + + it('should detect path regex', () => { + try { + // eslint-disable-next-line no-new + new UrlStorage( + { + type: 'url', + url: '../../myFolder/', + method: 'get' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the URL file: URL looks like a file path' + ) + }) +}) + +describe('URL Storage encryption integration tests', () => { + let storage: UrlStorage + let config: Awaited> + + before(async () => { + config = await getConfiguration() + storage = new UrlStorage( + { + type: 'url', + url: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', + method: 'get' + }, + config + ) + }) + + it('isEncrypted should return false for an unencrypted file', () => { + assert(storage.isEncrypted() === false, 'invalid response to isEncrypted()') + }) + + it('canDecrypt should return false when the file is not encrypted', () => { + assert( + storage.canDecrypt(nodeId) === false, + 'Wrong response from canDecrypt() for an unencrypted file' + ) + }) +}) + +describe('UrlStorage upload integration tests (Apache at 172.15.0.7:80)', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let config: Awaited> + let uploadStorage: UrlStorage + + before(async function () { + config = await getConfiguration() + uploadStorage = new UrlStorage( + { + type: 'url', + url: `${APACHE_BASE_URL}/`, + method: 'get' + }, + config + ) + }) + + it('upload sends PUT request and returns status', async function () { + const filename = `urlstorage-upload-test-${Date.now()}.txt` + const body = 'Hello from UrlStorage upload test' + const stream = Readable.from([body]) + const result = await uploadStorage.upload(filename, stream) + expect(result).to.have.property('httpStatus') + expect(result.httpStatus).to.be.oneOf([200, 201, 204]) + }) + + it('upload with url without trailing slash uses url as target', async function () { + const directUrl = `${APACHE_BASE_URL}/direct-put-${Date.now()}.txt` + const storageDirect = new UrlStorage( + { type: 'url', url: directUrl, method: 'get' }, + config + ) + const stream = Readable.from(['small payload']) + const result = await storageDirect.upload('ignored.txt', stream) + expect(result).to.have.property('httpStatus') + expect(result.httpStatus).to.be.oneOf([200, 201, 204]) + }) + + it('upload returns response headers', async function () { + const filename = `headers-test-${Date.now()}.txt` + const stream = Readable.from(['test']) + const result = await uploadStorage.upload(filename, stream) + expect(result).to.have.property('httpStatus') + expect(result).to.have.property('headers') + expect(typeof result.headers).to.equal('object') + }) +}) diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts deleted file mode 100644 index 394f7d737..000000000 --- a/src/test/unit/storage.test.ts +++ /dev/null @@ -1,660 +0,0 @@ -import { - Storage, - UrlStorage, - ArweaveStorage, - IpfsStorage -} from '../../components/storage/index.js' -import { - FileInfoRequest, - FileObjectType, - EncryptMethod -} from '../../@types/fileObject.js' -import { expect, assert } from 'chai' -import { - OverrideEnvConfig, - buildEnvOverrideConfig, - tearDownEnvironment, - setupEnvironment, - DEFAULT_TEST_TIMEOUT -} from '../utils/utils.js' -import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' -import { getConfiguration } from '../../utils/index.js' -import { expectedTimeoutFailure } from '../integration/testUtils.js' - -// let nodeId: string -const nodeId = '16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72' -const nodeId2 = '16Uiu2HAmQWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq73' -describe('URL Storage tests', () => { - let file: any = { - type: 'url', - url: 'http://someUrl.com/file.json', - method: 'get', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - }, - encryptedBy: nodeId, - encryptMethod: EncryptMethod.AES - } - let storage: UrlStorage - let error: Error - let config: any - before(async () => { - config = await getConfiguration() - storage = Storage.getStorageClass(file, config) as UrlStorage - }) - - it('Storage instance', () => { - expect(storage).to.be.instanceOf(UrlStorage) - }) - it('URL validation passes', () => { - expect(storage.validate()).to.eql([true, '']) - }) - it('isEncrypted should return true for an encrypted file', () => { - assert(storage.isEncrypted() === true, 'invalid response to isEncrypted()') - }) - - it('canDecrypt should return true for the correct nodeId', () => { - assert(storage.canDecrypt(nodeId) === true, "can't decrypt with the correct nodeId") - }) - - it('canDecrypt should return false for an incorrect nodeId', () => { - assert( - storage.canDecrypt('wrongNodeId') === false, - 'can decrypt with the wrong nodeId' - ) - }) - it('URL validation fails on missing URL', () => { - file = { - type: 'url', - method: 'get', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql( - 'Error validating the URL file: URL or method are missing' - ) - file = { - type: 'url', - url: 'http://someUrl.com/file.json', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql( - 'Error validating the URL file: URL or method are missing' - ) - }) - it('URL validation fails on invalid method', () => { - file = { - type: 'url', - url: 'http://someUrl.com/file.json', - method: 'put', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql('Error validating the URL file: Invalid method for URL') - }) - - it('URL validation fails on filename', () => { - file = { - type: 'url', - url: './../dir/file.json', - method: 'get', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql( - 'Error validating the URL file: URL looks like a file path' - ) - }) - it('Gets download URL', () => { - file = { - type: 'url', - url: 'http://someUrl.com/file.json', - method: 'get', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - storage = Storage.getStorageClass(file, config) as UrlStorage - expect(storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json') - }) - - it('Gets readable stream', async () => { - file = { - type: 'url', - url: 'https://stock-api.oceanprotocol.com/stock/stock.json', - method: 'get' - } - const storage = Storage.getStorageClass(file, config) - const stream = await storage.getReadableStream() - expect(stream).not.to.eql(null) - }) - - it('Gets readable stream with headers as plain object', async () => { - file = { - type: 'url', - url: 'https://stock-api.oceanprotocol.com/stock/stock.json', - method: 'get', - headers: { 'X-Test-Header': 'test' } - } - const storage = Storage.getStorageClass(file, config) - const stream = await storage.getReadableStream() - expect(stream).not.to.eql(null) - }) -}) - -describe('Unsafe URL tests', () => { - let previousConfiguration: OverrideEnvConfig[] - let file: any - let error: Error - let config: any - before(async () => { - previousConfiguration = await setupEnvironment( - null, - buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.UNSAFE_URLS], - [JSON.stringify(['^.*(169.254.169.254).*', '^.*(127.0.0.1).*'])] - ) - ) - config = await getConfiguration(true) - }) - - it('Should reject unsafe URL', () => { - file = { - type: 'url', - url: 'http://169.254.169.254/asfd', - method: 'get' - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql('Error validating the URL file: URL is marked as unsafe') - }) - it('Should allow safe URL', () => { - file = { - type: 'url', - url: 'https://oceanprotocol.com', - method: 'get' - } - const storage = Storage.getStorageClass(file, config) as UrlStorage - expect(storage.getDownloadUrl()).to.eql('https://oceanprotocol.com') - }) - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) - -describe('IPFS Storage tests', () => { - let file: any = { - type: 'ipfs', - hash: 'Qxchjkflsejdfklgjhfkgjkdjoiderj' - } - let error: Error - let previousConfiguration: OverrideEnvConfig[] - let config: any - before(async () => { - previousConfiguration = buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], - ['https://ipfs.oceanprotocol.com'] - ) - config = await getConfiguration() - }) - - it('Storage instance', () => { - expect(Storage.getStorageClass(file, config)).to.be.instanceOf(IpfsStorage) - }) - it('IPFS validation passes', () => { - expect(Storage.getStorageClass(file, config).validate()).to.eql([true, '']) - }) - it('IPFS validation fails', () => { - file = { - type: 'ipfs' - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql('Error validating the IPFS file: Missing CID') - }) - - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) - -describe('Arweave Storage tests', () => { - let file: any = { - type: 'arweave', - transactionId: '0x2563ed54abc0001bcaef' - } - - let error: Error - let previousConfiguration: OverrideEnvConfig[] - let config: any - - before(async () => { - previousConfiguration = buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.ARWEAVE_GATEWAY], - ['https://snaznabndfe3.arweave.net/nnLNdp6nuTb8mJ-qOgbUEx-9SBtBXQc_jejYOWzYEkM'] - ) - config = await getConfiguration() - }) - - it('Storage instance', () => { - expect(Storage.getStorageClass(file, config)).to.be.instanceOf(ArweaveStorage) - }) - it('Arweave validation passes', () => { - expect(Storage.getStorageClass(file, config).validate()).to.eql([true, '']) - }) - it('Arweave validation fails', () => { - file = { - type: 'arweave' - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql( - 'Error validating the Arweave file: Missing transaction ID' - ) - }) - - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) - -describe('URL Storage getFileInfo tests', () => { - let storage: UrlStorage - before(async () => { - const config = await getConfiguration() - storage = new UrlStorage( - { - type: 'url', - url: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', - method: 'get' - }, - config - ) - }) - - it('isEncrypted should return false for an encrypted file', () => { - assert(storage.isEncrypted() === false, 'invalid response to isEncrypted()') - }) - - it('canDecrypt should return false when the file is not encrypted', () => { - assert( - storage.canDecrypt('16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72') === - false, - 'Wrong response from canDecrypt() for an unencrypted file' - ) - }) - - it('Successfully retrieves file info for a URL', async () => { - const fileInfoRequest: FileInfoRequest = { - type: FileObjectType.URL - } - const fileInfo = await storage.getFileInfo(fileInfoRequest) - - assert(fileInfo[0].valid, 'File info is valid') - expect(fileInfo[0].contentLength).to.equal('319520') - expect(fileInfo[0].contentType).to.equal('text/plain; charset=utf-8') - expect(fileInfo[0].name).to.equal('shs_dataset_test.txt') - expect(fileInfo[0].type).to.equal('url') - }) - - it('Throws error when URL is missing in request', async () => { - const fileInfoRequest: FileInfoRequest = { type: FileObjectType.URL } - try { - await storage.getFileInfo(fileInfoRequest) - } catch (err) { - expect(err.message).to.equal('URL is required for type url') - } - }) -}) - -describe('URL Storage with malformed URL', () => { - let error: Error - let config: any - before(async () => { - config = await getConfiguration() - }) - - it('should detect path regex', () => { - try { - // eslint-disable-next-line no-new - new UrlStorage( - { - type: 'url', - url: '../../myFolder/', - method: 'get' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the URL file: URL looks like a file path' - ) - }) -}) - -describe('Arweave Storage getFileInfo tests', function () { - // this.timeout(15000) - let storage: ArweaveStorage - - before(async () => { - const config = await getConfiguration(true) - storage = new ArweaveStorage( - { - type: FileObjectType.ARWEAVE, - transactionId: 'gPPDyusRh2ZyFl-sQ2ODK6hAwCRBAOwp0OFKr0n23QE' - }, - config - ) - }) - - it('Successfully retrieves file info for an Arweave transaction', async () => { - const fileInfoRequest: FileInfoRequest = { - type: FileObjectType.ARWEAVE - } - const fileInfo = await storage.getFileInfo(fileInfoRequest) - - assert(fileInfo[0].valid, 'File info is valid') - assert(fileInfo[0].type === FileObjectType.ARWEAVE, 'Type is incorrect') - assert( - fileInfo[0].contentType === 'text/csv; charset=utf-8', - 'Content type is incorrect' - ) - assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect') - }) - - it('Throws error when transaction ID is missing in request', async () => { - const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE } - try { - await storage.getFileInfo(fileInfoRequest) - } catch (err) { - expect(err.message).to.equal('Transaction ID is required for type arweave') - } - }) -}) - -describe('Arweave Storage with malformed transaction ID', () => { - let error: Error - let config: any - before(async () => { - config = await getConfiguration() - }) - it('should detect URL path format', () => { - try { - // eslint-disable-next-line no-new - new ArweaveStorage( - { - type: 'arweave', - transactionId: - 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the Arweave file: Transaction ID looks like an URL. Please specify URL storage instead.' - ) - }) - - it('should detect path regex', () => { - try { - // eslint-disable-next-line no-new - new ArweaveStorage( - { - type: 'arweave', - transactionId: '../../myFolder/' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the Arweave file: Transaction ID looks like a file path' - ) - }) -}) - -describe('Arweave Storage with malformed transaction ID', () => { - let error: Error - let config: any - before(async () => { - config = await getConfiguration() - }) - it('should detect URL path format', () => { - try { - // eslint-disable-next-line no-new - new IpfsStorage( - { - type: 'ipfs', - hash: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the IPFS file: CID looks like an URL. Please specify URL storage instead.' - ) - }) - - it('should detect path regex', () => { - try { - // eslint-disable-next-line no-new - new IpfsStorage( - { - type: 'ipfs', - hash: '../../myFolder/' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the IPFS file: CID looks like a file path' - ) - }) -}) - -describe('IPFS Storage getFileInfo tests', function () { - let storage: IpfsStorage - let previousConfiguration: OverrideEnvConfig[] - let config: any - before(async () => { - previousConfiguration = await buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], - ['https://ipfs.oceanprotocol.com'] - ) - await setupEnvironment(undefined, previousConfiguration) // Apply the environment override - config = await getConfiguration() - - storage = new IpfsStorage( - { - type: FileObjectType.IPFS, - hash: 'QmRhsp7eghZtW4PktPC2wAHdKoy2LiF1n6UXMKmAhqQJUA' - }, - config - ) - }) - - it('Successfully retrieves file info for an IPFS hash', function () { - // this test fails often because of timeouts apparently - // so we increase the deafult timeout - this.timeout(DEFAULT_TEST_TIMEOUT * 2) - const fileInfoRequest: FileInfoRequest = { - type: FileObjectType.IPFS - } - // and only fire the test half way - setTimeout(async () => { - const fileInfo = await storage.getFileInfo(fileInfoRequest) - if (fileInfo && fileInfo.length > 0) { - assert(fileInfo[0].valid, 'File info is valid') - assert(fileInfo[0].type === 'ipfs', 'Type is incorrect') - // if these are not available is because we could not fetch the metadata yet - if (fileInfo[0].contentType && fileInfo[0].contentLength) { - assert(fileInfo[0].contentType === 'text/csv', 'Content type is incorrect') - assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect') - } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(true) - } - }, DEFAULT_TEST_TIMEOUT) - }) - - it('Throws error when hash is missing in request', async () => { - const fileInfoRequest: FileInfoRequest = { type: FileObjectType.IPFS } - try { - await storage.getFileInfo(fileInfoRequest) - } catch (err) { - expect(err.message).to.equal('Hash is required for type ipfs') - } - }) - - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) - -describe('URL Storage encryption tests', () => { - let storage: UrlStorage - let config: any - before(async () => { - config = await getConfiguration() - storage = new UrlStorage( - { - type: 'url', - url: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', - method: 'get' - }, - config - ) - }) - - it('isEncrypted should return false for an encrypted file', () => { - assert(storage.isEncrypted() === false, 'invalid response to isEncrypted()') - }) - - it('canDecrypt should return false when the file is not encrypted', () => { - assert( - storage.canDecrypt(nodeId) === false, - 'Wrong response from canDecrypt() for an unencrypted file' - ) - }) -}) - -describe('URL Storage encryption tests', function () { - this.timeout(15000) - let storage: IpfsStorage - let previousConfiguration: OverrideEnvConfig[] - - before(async () => { - previousConfiguration = await buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], - ['https://ipfs.oceanprotocol.com'] - ) - await setupEnvironment(undefined, previousConfiguration) // Apply the environment override - const config = await getConfiguration() - storage = new IpfsStorage( - { - type: 'ipfs', - hash: 'QmQVPuoXMbVEk7HQBth5pGPPMcgvuq4VSgu2XQmzU5M2Pv', - encryptedBy: nodeId, - encryptMethod: EncryptMethod.AES - }, - config - ) - }) - - it('isEncrypted should return true for an encrypted file', () => { - assert(storage.isEncrypted() === true, 'invalid response to isEncrypted()') - }) - - it('canDecrypt should return true for this node', () => { - assert( - storage.canDecrypt(nodeId) === true, - 'Wrong response from canDecrypt() for an encrypted file' - ) - }) - - it('File info includes encryptedBy and encryptMethod', function () { - // same thing here, IFPS takes time - this.timeout(DEFAULT_TEST_TIMEOUT * 2) - const fileInfoRequest: FileInfoRequest = { - type: FileObjectType.IPFS - } - - setTimeout(async () => { - const fileInfo = await storage.getFileInfo(fileInfoRequest) - if (fileInfo && fileInfo.length > 0) { - assert(fileInfo[0].valid, 'File info is valid') - expect(fileInfo[0].type).to.equal('ipfs') - - // same thing as above, these tests should consider that the metadata exists, - // its not on our side anyway - if (fileInfo[0].contentType && fileInfo[0].encryptedBy) { - expect(fileInfo[0].contentType).to.equal('application/octet-stream') - expect(fileInfo[0].encryptedBy).to.equal(nodeId) - expect(fileInfo[0].encryptMethod).to.equal(EncryptMethod.AES) - } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(true) - } - }, DEFAULT_TEST_TIMEOUT) - }) - - it('canDecrypt should return false when called from an unauthorised node', () => { - assert( - storage.canDecrypt(nodeId) === true, - 'Wrong response from canDecrypt() for an unencrypted file' - ) - assert( - storage.canDecrypt(nodeId2) === false, - 'Wrong response from canDecrypt() for an unencrypted file' - ) - }) - - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) diff --git a/src/utils/config/schemas.ts b/src/utils/config/schemas.ts index 15671d9c7..7246e8de9 100644 --- a/src/utils/config/schemas.ts +++ b/src/utils/config/schemas.ts @@ -118,7 +118,10 @@ export const ComputeResourceSchema = z.object({ min: z.number().optional(), max: z.number().optional(), inUse: z.number().optional(), - init: z.any().optional() + init: z.any().optional(), + platform: z.string().optional(), + memoryTotal: z.string().optional(), + driverVersion: z.string().optional() }) export const ComputeResourcesPricingInfoSchema = z.object({