diff --git a/.tmp_gateway_storage_layout.json b/.tmp_gateway_storage_layout.json deleted file mode 100644 index 0b9d4872c7..0000000000 --- a/.tmp_gateway_storage_layout.json +++ /dev/null @@ -1,7 +0,0 @@ - -╭------+----------------------------+------+--------+-------+---------------------------------------------╮ -| Name | Type | Slot | Offset | Bytes | Contract | -+=========================================================================================================+ -| s | struct GatewayActorStorage | 0 | 0 | 1056 | contracts/GatewayDiamond.sol:GatewayDiamond | -╰------+----------------------------+------+--------+-------+---------------------------------------------╯ - diff --git a/.tmp_gateway_storage_layout.txt b/.tmp_gateway_storage_layout.txt deleted file mode 100644 index 0b9d4872c7..0000000000 --- a/.tmp_gateway_storage_layout.txt +++ /dev/null @@ -1,7 +0,0 @@ - -╭------+----------------------------+------+--------+-------+---------------------------------------------╮ -| Name | Type | Slot | Offset | Bytes | Contract | -+=========================================================================================================+ -| s | struct GatewayActorStorage | 0 | 0 | 1056 | contracts/GatewayDiamond.sol:GatewayDiamond | -╰------+----------------------------+------+--------+-------+---------------------------------------------╯ - diff --git a/Cargo.lock b/Cargo.lock index e375d903a3..c07e7cca66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7289,15 +7289,19 @@ dependencies = [ "async-channel 1.9.0", "async-trait", "base64 0.21.7", + "blake3", "bytes", "chrono", "cid 0.11.1", "clap 4.5.50", "clap_complete", "contracts-artifacts", + "dirs", "env_logger 0.10.2", "ethers", "ethers-contract", + "fendermint_actor_blobs_shared", + "fendermint_actor_bucket", "fendermint_app", "fendermint_app_settings", "fendermint_crypto", @@ -7306,6 +7310,7 @@ dependencies = [ "fendermint_eth_hardhat", "fendermint_rpc", "fendermint_vm_actor_interface", + "fendermint_vm_message", "fil_actors_runtime", "flate2", "fs-err", @@ -7321,6 +7326,7 @@ dependencies = [ "ipc-wallet", "ipc_actors_abis", "ipc_ipld_resolver", + "iroh-blobs", "libp2p", "libsecp256k1", "log", @@ -7354,6 +7360,7 @@ dependencies = [ "url", "urlencoding", "uuid 1.18.1", + "walkdir", "warp", "zeroize", ] @@ -11810,6 +11817,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "native-tls", "once_cell", "percent-encoding", diff --git a/calculate_chain_id.py b/calculate_chain_id.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/fendermint/vm/interpreter/src/fvm/interpreter.rs b/fendermint/vm/interpreter/src/fvm/interpreter.rs index 59cdb8f964..459d8a1c6a 100644 --- a/fendermint/vm/interpreter/src/fvm/interpreter.rs +++ b/fendermint/vm/interpreter/src/fvm/interpreter.rs @@ -277,12 +277,12 @@ where "prepare_messages_for_block decoded mempool messages" ); - let signed_msgs = - select_messages_above_base_fee(signed_msgs, state.block_gas_tracker().base_fee()); - tracing::info!( - selected_by_base_fee = signed_msgs.len(), - "prepare_messages_for_block selected messages above base fee" - ); + // let signed_msgs = + // select_messages_above_base_fee(signed_msgs, state.block_gas_tracker().base_fee()); + // tracing::info!( + // selected_by_base_fee = signed_msgs.len(), + // "prepare_messages_for_block selected messages above base fee" + // ); let total_gas_limit = state.block_gas_tracker().available(); let signed_msg_count = signed_msgs.len(); diff --git a/ipc-storage/README.md b/ipc-storage/README.md index 5d76012edd..afdcc9b978 100644 --- a/ipc-storage/README.md +++ b/ipc-storage/README.md @@ -1,31 +1,36 @@ -# Bucket Storage Guide (Path-Based Access) +# Bucket Storage Guide -## Configuration - -```bash -# From RECALL_RUN.md -export TENDERMINT_RPC=http://localhost:26657 -export OBJECTS_LISTEN_ADDR=http://localhost:8080 -export NODE_OPERATION_OBJECT_API=http://localhost:8081 -export ETH_RPC=http://localhost:8545 -export BLOBS_ACTOR=0x6d342defae60f6402aee1f804653bbae4e66ae46 -export ADM_ACTOR=0x7caec36fc8a3a867ca5b80c6acb5e5871d05aa28 +## 1. Build IPC -# Your credentials -export USER_SK= -``` +By default, ipc-storage is not enabled. Build with the `ipc-storage` feature: -## 1. Build IPC -By default, ipc-storage is not enabled, build with `ipc-storage` feature to enable it. ```bash cargo build --release -p fendermint_app --features ipc-storage +cargo build --release -p ipc-cli --features ipc-storage +cargo build --release -p ipc-decentralized-storage --bin gateway --bin node ``` -Setup your ipc chain as per normal. -## 2. Start Gateway and Node Operator +Set up your IPC chain as per normal. + +## 2. Initialize Storage Config + ```bash -cargo build --release -p ipc-decentralized-storage --bin gateway --bin node +./target/release/ipc-cli storage init +``` +This generates `~/.ipc/storage.yaml` with defaults. Update it if needed: + +```yaml +# Key fields you may want to adjust: +secret-key-file: ./test-network/keys/alice.sk # your funded key +gateway-url: http://127.0.0.1:8080 # gateway address +tendermint-rpc-url: http://127.0.0.1:26657 +eth-rpc-url: http://127.0.0.1:8545 +``` + +## 3. Start Gateway and Node Operator + +```bash # prepare to start node export FM_NETWORK=test # validator bls key file in hex format @@ -47,218 +52,54 @@ export SECRET_KEY_FILE=./test-network/keys/alice.sk --max-concurrent-downloads 10 \ --rpc-bind-addr 127.0.0.1:8081 -./target/release/gateway --bls-key-file $BLS_KEY_FILE --secret-key-file $SECRET_KEY_FILE --iroh-path ./iroh_gateway --objects-listen-addr 0.0.0.0:8080 - -``` - -## 3. Launch ipc-dropbox -Launch `ipc-dropbox` in `ipc-storage/ipc-dropbox` with `npm run dev`. - -Alternatively, run the below bash script to go through the steps one by one. - -## 3. Create a Bucket - -First, create a bucket via the ADM (Actor Deployment Manager): - -```bash -# Buy 1 FIL worth of credits -cast send $BLOBS_ACTOR "buyCredit()" \ - --value 0.1ether \ - --private-key $USER_SK \ - --rpc-url http://localhost:8545 - -# Create a new bucket (caller becomes owner) -TX_RESULT=$(cast send $ADM_ACTOR "createBucket()" \ - --private-key $USER_SK \ - --rpc-url $ETH_RPC \ - --json) - -echo $TX_RESULT | jq '.' - -# Extract bucket address from MachineInitialized event -# Event signature: MachineInitialized(uint8 indexed kind, address machineAddress) -BUCKET_ADDR=$(echo $TX_RESULT | jq -r '.logs[] | select(.topics[0] == "0x8f7252642373d5f0b89a0c5cd9cd242e5cd5bb1a36aec623756e4f52a8c1ea6e") | .data' | cut -c27-66) -BUCKET_ADDR="0x$BUCKET_ADDR" - -echo "Bucket created at: $BUCKET_ADDR" -export BUCKET_ADDR -``` - -## 4. Upload and Register an Object -```bash -# Create a test file -echo "Hello from bucket storage!" > myfile.txt - -# Get file size -BLOB_SIZE=$(stat -f%z myfile.txt 2>/dev/null || stat -c%s myfile.txt) - -# Upload to Iroh -UPLOAD_RESPONSE=$(curl -s -X POST $OBJECTS_API/v1/objects \ - -F "size=${BLOB_SIZE}" \ - -F "data=@myfile.txt") - -echo $UPLOAD_RESPONSE | jq '.' - -# Extract hashes -BLOB_HASH_B32=$(echo $UPLOAD_RESPONSE | jq -r '.hash') -METADATA_HASH_B32=$(echo $UPLOAD_RESPONSE | jq -r '.metadata_hash // .metadataHash') -NODE_ID_BASE32=$(curl -s $OBJECTS_API/v1/node | jq -r '.node_id') - -# Convert to hex (same as RECALL_RUN.md) -export BLOB_HASH=$(python3 -c " -import base64 -h = '$BLOB_HASH_B32'.upper() -padding = (8 - len(h) % 8) % 8 -h = h + '=' * padding -decoded = base64.b32decode(h) -if len(decoded) > 32: - decoded = decoded[:32] -elif len(decoded) < 32: - decoded = decoded + b'\x00' * (32 - len(decoded)) -print('0x' + decoded.hex()) -") - -export METADATA_HASH=$(python3 -c " -import base64 -h = '$METADATA_HASH_B32'.upper() -padding = (8 - len(h) % 8) % 8 -h = h + '=' * padding -decoded = base64.b32decode(h) -if len(decoded) > 32: - decoded = decoded[:32] -elif len(decoded) < 32: - decoded = decoded + b'\x00' * (32 - len(decoded)) -print('0x' + decoded.hex()) -") - -export SOURCE_NODE="0x$NODE_ID_BASE32" - -echo "Blob Hash: $BLOB_HASH" -echo "Metadata Hash: $METADATA_HASH" -echo "Source Node: $SOURCE_NODE" +./target/release/gateway --bls-key-file $BLS_KEY_FILE --secret-key-file $SECRET_KEY_FILE --iroh-path ./iroh_gateway --objects-listen-addr 127.0.0.1:8080 ``` -### Register object in bucket with a path - -```bash -# Add object with a path-based key -# Signature: addObject(bytes32 source, string key, bytes32 hash, bytes32 recoveryHash, uint64 size) -cast send $BUCKET_ADDR "addObject(bytes32,string,bytes32,bytes32,uint64)" \ - $SOURCE_NODE \ - "documents/myfile.txt" \ - $BLOB_HASH \ - $METADATA_HASH \ - $BLOB_SIZE \ - --private-key $USER_SK \ - --rpc-url $ETH_RPC -``` - -## 5. Query Objects - -### Get a single object by path +## 4. Buy Credits ```bash -# Get object by exact path -# Returns: ObjectValue(bytes32 blobHash, bytes32 recoveryHash, uint64 size, uint64 expiry, (string,string)[] metadata) -cast call $BUCKET_ADDR "getObject(string)((bytes32,bytes32,uint64,uint64,(string,string)[]))" "documents/myfile.txt" --rpc-url $ETH_RPC +./target/release/ipc-cli storage credit buy 0.1 ``` -### List all objects (no filter) +## 5. Create a Bucket ```bash -# List all objects in bucket -cast call $BUCKET_ADDR "queryObjects()(((string,(bytes32,uint64,uint64,(string,string)[]))[],string[],string))" \ - --rpc-url $ETH_RPC +./target/release/ipc-cli storage bucket create ``` -### List with prefix (folder-like) +This prints the bucket addresses (Actor ID, EVM, robust). Export the address for later use: ```bash -# List everything under "documents/" -cast call $BUCKET_ADDR "queryObjects(string)(((string,(bytes32,uint64,uint64,(string,string)[]))[],string[],string))" "documents/" --rpc-url $ETH_RPC +export BUCKET_ADDR=t065 # use the actor ID from the output ``` -### List with delimiter (S3-style folder simulation) +## 6. Upload Files ```bash -# List top-level "folders" and files -# Returns: Query((string,ObjectState)[] objects, string[] commonPrefixes, string nextKey) -# Where ObjectState = (bytes32 blobHash, uint64 size, uint64 expiry, (string,string)[] metadata) -cast call $BUCKET_ADDR "queryObjects(string,string)(((string,(bytes32,uint64,uint64,(string,string)[]))[],string[],string))" "" "/" \ - --rpc-url $ETH_RPC - -# Example response: -# ([], ["documents/", "images/"], "") -# ^objects at root ^"folders" ^nextKey (empty = no more pages) - -# Extract blob hash from first object: -# BLOB_HASH=$(cast call ... | jq -r '.[0][0][1][0]') - -# List contents of "documents/" folder -cast call $BUCKET_ADDR "queryObjects(string,string)(((string,(bytes32,uint64,uint64,(string,string)[]))[],string[],string))" "documents/" "/" \ - --rpc-url $ETH_RPC +# Upload a single file +echo "Hello from bucket storage!" > myfile.txt +./target/release/ipc-cli storage cp ./myfile.txt "ipc://${BUCKET_ADDR}/documents/myfile.txt" ``` -### Paginated queries +## 7. Query Objects ```bash -# Query with pagination -# queryObjects(prefix, delimiter, startKey, limit) -cast call $BUCKET_ADDR "queryObjects(string,string,string,uint64)" \ - "documents/" \ - "/" \ - "" \ - 100 \ - --rpc-url $ETH_RPC - -# If nextKey is returned, use it for the next page -cast call $BUCKET_ADDR "queryObjects(string,string,string,uint64)" \ - "documents/" \ - "/" \ - "documents/page2start.txt" \ - 100 \ - --rpc-url $ETH_RPC -``` - ---- +# List all objects in a bucket +./target/release/ipc-cli storage ls "ipc://${BUCKET_ADDR}/" -## 6. Update Object Metadata - -```bash -# Update metadata for an existing object -# Set value to empty string to delete a metadata key -cast send $BUCKET_ADDR "updateObjectMetadata(string,(string,string)[])" \ - "documents/myfile.txt" \ - '[("content-type","text/markdown"),("version","2")]' \ - --private-key $USER_SK \ - --rpc-url $ETH_RPC +# Get object metadata +./target/release/ipc-cli storage stat "ipc://${BUCKET_ADDR}/documents/myfile.txt" ``` ---- - -## 7. Delete an Object +## 8. Read File Contents ```bash -# Delete object by path -cast send $BUCKET_ADDR "deleteObject(string)" "documents/myfile.txt" \ - --private-key $USER_SK \ - --rpc-url $ETH_RPC +./target/release/ipc-cli storage cat "ipc://${BUCKET_ADDR}/documents/myfile.txt" ``` ---- - -## 8. Download Content - -Downloads still go through the Iroh/Objects API using the blob hash: +## 9. Download Files ```bash -# First get the object to retrieve its blob hash -OBJECT_INFO=$(cast call $BUCKET_ADDR "getObject(string)" "documents/myfile.txt" \ - --rpc-url $ETH_RPC) - -# Extract blob hash from response and download -# (The blob hash is the first bytes32 in the response) -curl $NODE_OPERATION_OBJECT_API/v1/blobs/${BLOB_HASH#0x}/content +# Download a single file +./target/release/ipc-cli storage cp "ipc://${BUCKET_ADDR}/documents/myfile.txt" ./downloaded.txt ``` - ---- \ No newline at end of file diff --git a/ipc/cli/Cargo.toml b/ipc/cli/Cargo.toml index f0ac0133e9..c5c3ecffe8 100644 --- a/ipc/cli/Cargo.toml +++ b/ipc/cli/Cargo.toml @@ -4,6 +4,10 @@ version = "0.1.0" edition.workspace = true license-file.workspace = true +[features] +default = [] +ipc-storage = ["fendermint_app/ipc-storage"] + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [build-dependencies] @@ -40,7 +44,7 @@ num-derive = "0.3.3" num-bigint = { workspace = true } num-traits = { workspace = true } openssl = { workspace = true } -reqwest = { workspace = true } +reqwest = { workspace = true, features = ["multipart"] } serde = { workspace = true } serde_bytes = "0.11.9" serde_json = { workspace = true } @@ -68,7 +72,7 @@ uuid = { version = "1.0", features = ["v4"] } mime_guess = "2.0" include_dir = "0.7" fendermint_eth_api = { path = "../../fendermint/eth/api" } -fendermint_rpc = { path = "../../fendermint/rpc" } +fendermint_rpc = { path = "../../fendermint/rpc", features = ["ipc-storage"] } tendermint-rpc = { workspace = true } ipc-wallet = { path = "../../ipc/wallet", features = ["with-ethers"] } @@ -82,8 +86,15 @@ fendermint_eth_hardhat = { path = "../../fendermint/eth/hardhat" } fendermint_eth_deployer = { path = "../../fendermint/eth/deployer" } fendermint_app_settings = { path = "../../fendermint/app/settings" } fendermint_vm_actor_interface = { path = "../../fendermint/vm/actor_interface" } +fendermint_vm_message = { path = "../../fendermint/vm/message" } +fendermint_actor_blobs_shared = { path = "../../fendermint/actors/blobs/shared" } +fendermint_actor_bucket = { path = "../../fendermint/actors/bucket" } fendermint_app = { path = "../../fendermint/app" } fendermint_crypto = { path = "../../fendermint/crypto" } ipc_ipld_resolver = { path = "../../ipld/resolver" } contracts-artifacts = { path = "../../contracts-artifacts" } ipc_actors_abis = { path = "../../contract-bindings" } +walkdir = "2.4" +dirs = "5.0" +blake3 = "1.5" +iroh-blobs = { workspace = true } diff --git a/ipc/cli/src/commands/mod.rs b/ipc/cli/src/commands/mod.rs index 1fd0128a27..22c836068c 100644 --- a/ipc/cli/src/commands/mod.rs +++ b/ipc/cli/src/commands/mod.rs @@ -11,6 +11,7 @@ mod node; mod subnet; mod ui; mod util; +mod storage; mod validator; mod wallet; @@ -39,6 +40,7 @@ use std::str::FromStr; use crate::commands::config::ConfigCommandsArgs; use crate::commands::deploy::{DeployCommand, DeployCommandArgs}; use crate::commands::node::NodeCommandsArgs; +use crate::commands::storage::StorageCommandsArgs; use crate::commands::validator::ValidatorCommandsArgs; use crate::commands::wallet::WalletCommandsArgs; use crate::CommandLineHandler; @@ -62,6 +64,7 @@ enum Commands { Deploy(DeployCommandArgs), Ui(UICommandArgs), Node(NodeCommandsArgs), + Storage(StorageCommandsArgs), } #[derive(Debug, Parser)] @@ -157,6 +160,7 @@ pub async fn cli() -> anyhow::Result<()> { Commands::Deploy(args) => DeployCommand::handle(global, args).await, Commands::Ui(args) => run_ui_command(global.clone(), args.clone()).await, Commands::Node(args) => args.handle(global).await, + Commands::Storage(args) => args.handle(global).await, }; r.with_context(|| format!("error processing command {:?}", args.command)) diff --git a/ipc/cli/src/commands/storage/bucket.rs b/ipc/cli/src/commands/storage/bucket.rs new file mode 100644 index 0000000000..a8f0d448e0 --- /dev/null +++ b/ipc/cli/src/commands/storage/bucket.rs @@ -0,0 +1,305 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Bucket operations for on-chain storage management +//! +//! This module provides functions to interact with bucket smart contracts. + +use anyhow::{anyhow, Context, Result}; +use fendermint_actor_bucket::{ + AddParams, DeleteParams, GetParams, ListObjectsReturn, ListParams, Object, UpdateObjectMetadataParams, + Method as BucketMethod, +}; +use fendermint_actor_blobs_shared::bytes::B256; +use fendermint_rpc::{message::GasParams, tx::{BoundClient, TxClient, TxCommit}, QueryClient}; +use fendermint_vm_message::query::FvmQueryHeight; +use fvm_ipld_encoding::RawBytes; +use fvm_shared::{address::Address, chainid::ChainID, econ::TokenAmount}; +use num_traits::Zero; +use std::collections::HashMap; + +/// Default gas parameters for bucket transactions +fn default_gas_params() -> GasParams { + GasParams { + gas_limit: 10_000_000_000, + gas_fee_cap: TokenAmount::from_atto(10_000), + gas_premium: TokenAmount::from_atto(10_000), + } +} + +/// Add an object to a bucket +/// +/// This registers an object's metadata on-chain after the blob has been uploaded +/// to the gateway and distributed to storage nodes. +pub async fn add_object( + client: &mut C, + bucket_address: Address, + source: B256, + key: String, + hash: B256, + recovery_hash: B256, + size: u64, + metadata: HashMap, + data_shards: u16, + parity_shards: u16, +) -> Result<()> +where + C: BoundClient + TxClient + Send + Sync, +{ + let params = AddParams { + source, + key: key.into_bytes(), + hash, + recovery_hash, + size, + ttl: None, // Use default TTL + metadata, + overwrite: false, + data_shards, + parity_shards, + }; + + let params_bytes = RawBytes::serialize(params).context("Failed to serialize AddParams")?; + + let res = TxClient::::transaction( + client, + bucket_address, + BucketMethod::AddObject as u64, + params_bytes, + TokenAmount::zero(), + default_gas_params(), + ) + .await + .context("Failed to send AddObject transaction")?; + + if res.response.check_tx.code.is_err() { + let log = &res.response.check_tx.log; + let info = &res.response.check_tx.info; + return Err(anyhow!( + "AddObject check_tx failed (code {:?}): log={} info={}", + res.response.check_tx.code, + if log.is_empty() { "" } else { log }, + if info.is_empty() { "" } else { info }, + )); + } + + if res.response.deliver_tx.code.is_err() { + let log = &res.response.deliver_tx.log; + let info = &res.response.deliver_tx.info; + return Err(anyhow!( + "AddObject deliver_tx failed (code {:?}): log={} info={}", + res.response.deliver_tx.code, + if log.is_empty() { "" } else { log }, + if info.is_empty() { "" } else { info }, + )); + } + + Ok(()) +} + +/// Get an object from a bucket +pub async fn get_object( + client: &mut C, + bucket_address: Address, + key: String, +) -> Result> +where + C: QueryClient + Send + Sync, +{ + let params = GetParams(key.into_bytes()); + let gas_params = GasParams { + gas_limit: Default::default(), + gas_fee_cap: Default::default(), + gas_premium: Default::default(), + }; + + client + .os_get_call( + bucket_address, + params, + TokenAmount::default(), + gas_params, + FvmQueryHeight::default(), + ) + .await + .context("Failed to query object") +} + +/// List objects in a bucket +pub async fn list_objects( + client: &C, + bucket_address: Address, + prefix: Option, + delimiter: Option, + start_key: Option, + limit: u64, +) -> Result +where + C: QueryClient + Send + Sync, +{ + let params = ListParams { + prefix: prefix.unwrap_or_default().into_bytes(), + delimiter: delimiter.unwrap_or_default().into_bytes(), + start_key: start_key.map(|s| s.into_bytes()), + limit, + }; + + let params_bytes = RawBytes::serialize(params).context("Failed to serialize ListParams")?; + + let msg = fvm_shared::message::Message { + version: Default::default(), + from: fendermint_vm_actor_interface::system::SYSTEM_ACTOR_ADDR, + to: bucket_address, + sequence: 0, + value: TokenAmount::zero(), + method_num: BucketMethod::ListObjects as u64, + params: params_bytes, + gas_limit: 10_000_000_000, + gas_fee_cap: TokenAmount::zero(), + gas_premium: TokenAmount::zero(), + }; + + let response = client + .call(msg, FvmQueryHeight::default()) + .await + .context("Failed to execute ListObjects call")?; + + if response.value.code.is_err() { + return Err(anyhow!("ListObjects query failed: {}", response.value.info)); + } + + let return_data = fendermint_rpc::response::decode_data(&response.value.data) + .context("Failed to decode response data")?; + + let result = fvm_ipld_encoding::from_slice::(&return_data) + .context("Failed to decode ListObjects response")?; + + Ok(result) +} + +/// Delete an object from a bucket +pub async fn delete_object( + client: &mut C, + bucket_address: Address, + key: String, +) -> Result<()> +where + C: BoundClient + TxClient + Send + Sync, +{ + let params = DeleteParams(key.into_bytes()); + let params_bytes = RawBytes::serialize(params).context("Failed to serialize DeleteParams")?; + + let res = TxClient::::transaction( + client, + bucket_address, + BucketMethod::DeleteObject as u64, + params_bytes, + TokenAmount::zero(), + default_gas_params(), + ) + .await + .context("Failed to send DeleteObject transaction")?; + + if res.response.check_tx.code.is_err() { + return Err(anyhow!( + "DeleteObject check_tx failed: {}", + res.response.check_tx.log + )); + } + + if res.response.deliver_tx.code.is_err() { + return Err(anyhow!( + "DeleteObject deliver_tx failed: {}", + res.response.deliver_tx.log + )); + } + + Ok(()) +} + +/// Query the chain ID from the network +pub async fn query_chain_id(client: &C) -> Result +where + C: QueryClient + Send + Sync, +{ + let state_params = client + .state_params(FvmQueryHeight::default()) + .await + .context("Failed to query state params for chain ID")?; + + Ok(ChainID::from(state_params.value.chain_id)) +} + +/// Convert a hex string to a B256, with length validation. +/// +/// Accepts with or without "0x" prefix. Returns an error if the decoded +/// bytes are not exactly 32 bytes long. +pub fn hex_to_b256(hex_str: &str) -> Result { + let hex_str = hex_str.strip_prefix("0x").unwrap_or(hex_str); + let bytes = hex::decode(hex_str).context("Invalid hex string")?; + if bytes.len() != 32 { + return Err(anyhow!( + "Expected 32 bytes, got {} bytes from hex string", + bytes.len() + )); + } + let mut array = [0u8; 32]; + array.copy_from_slice(&bytes); + Ok(B256(array)) +} + +/// Convert a hash string to B256, auto-detecting hex or base32 encoding. +/// +/// Supports: +/// - Hex with "0x" prefix +/// - Hex (64 hex chars) +/// - Base32 lower-case no-padding (iroh/blake3 format, 52 chars) +pub fn hash_to_b256(s: &str) -> Result { + if s.starts_with("0x") || (s.len() == 64 && s.chars().all(|c| c.is_ascii_hexdigit())) { + return hex_to_b256(s); + } + // Try base32 (lower-case no-padding, as used by iroh) + let bytes = base32_decode_nopad(s).context("Failed to decode as base32")?; + if bytes.len() < 32 { + return Err(anyhow!( + "Expected at least 32 bytes, got {} from base32 string", + bytes.len() + )); + } + let mut array = [0u8; 32]; + array.copy_from_slice(&bytes[..32]); + Ok(B256(array)) +} + +/// Decode RFC 4648 base32 (case-insensitive, no padding required). +fn base32_decode_nopad(input: &str) -> Result> { + const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + + fn val(c: u8) -> Result { + let c = c.to_ascii_uppercase(); + ALPHABET + .iter() + .position(|&a| a == c) + .map(|p| p as u8) + .ok_or_else(|| anyhow!("invalid base32 character: {}", c as char)) + } + + let input = input.as_bytes(); + let mut buf = Vec::with_capacity(input.len() * 5 / 8); + let mut bits: u32 = 0; + let mut n_bits: u32 = 0; + + for &c in input { + if c == b'=' { + break; + } + bits = (bits << 5) | val(c)? as u32; + n_bits += 5; + if n_bits >= 8 { + n_bits -= 8; + buf.push((bits >> n_bits) as u8); + bits &= (1 << n_bits) - 1; + } + } + Ok(buf) +} diff --git a/ipc/cli/src/commands/storage/bucket_cmd.rs b/ipc/cli/src/commands/storage/bucket_cmd.rs new file mode 100644 index 0000000000..e919d940fe --- /dev/null +++ b/ipc/cli/src/commands/storage/bucket_cmd.rs @@ -0,0 +1,341 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Bucket subcommand for creating and managing storage buckets on-chain. + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use clap::{Args, Subcommand}; +use num_traits::Zero; +use std::collections::HashMap; +use std::path::PathBuf; + +use fendermint_rpc::client::FendermintClient; +use fendermint_rpc::message::{GasParams, SignedMessageFactory}; +use fendermint_rpc::tx::{TxClient, TxCommit}; +use fendermint_rpc::QueryClient; +use fendermint_vm_actor_interface::adm::{ + self, CreateExternalParams, CreateExternalReturn, Kind, ListMetadataParams, + Method as AdmMethod, +}; +use fendermint_vm_actor_interface::eam::EthAddress; +use fendermint_vm_message::query::FvmQueryHeight; +use fvm_ipld_encoding::RawBytes; +use fvm_shared::address::Address; +use fvm_shared::chainid::ChainID; +use fvm_shared::econ::TokenAmount; + +use crate::commands::storage::config::StorageConfig; +use crate::{CommandLineHandler, GlobalArguments}; + +#[derive(Debug, Args)] +#[command(name = "bucket", about = "Create and manage storage buckets")] +pub struct BucketCommandArgs { + #[command(subcommand)] + command: BucketCommands, +} + +#[derive(Debug, Subcommand)] +pub enum BucketCommands { + /// Create a new storage bucket + Create(CreateBucketArgs), + /// List buckets owned by an address + List(ListBucketsArgs), +} + +impl BucketCommandArgs { + pub async fn handle(&self, global: &GlobalArguments) -> anyhow::Result<()> { + match &self.command { + BucketCommands::Create(args) => CreateBucket::handle(global, args).await, + BucketCommands::List(args) => ListBuckets::handle(global, args).await, + } + } +} + +// --------------------------------------------------------------------------- +// Create +// --------------------------------------------------------------------------- + +#[derive(Debug, Args)] +pub struct CreateBucketArgs { + /// Storage config file + #[arg(long)] + pub config: Option, + + /// Optional owner address (defaults to the operator key address) + #[arg(long)] + pub owner: Option, + + /// Optional metadata key=value pairs (can be repeated) + #[arg(long = "metadata", value_name = "KEY=VALUE")] + pub metadata: Vec, +} + +pub struct CreateBucket; + +#[async_trait] +impl CommandLineHandler for CreateBucket { + type Arguments = CreateBucketArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + let fm_client = FendermintClient::new_http(config.tendermint_rpc_url.parse()?, None)?; + + let chain_id = super::bucket::query_chain_id(&fm_client) + .await + .context("Failed to query chain ID")?; + + let secret_key = + SignedMessageFactory::read_secret_key(&config.secret_key_file)?; + let pub_key = secret_key.public_key(); + let addr = Address::new_secp256k1(&pub_key.serialize())?; + + // Parse owner address — ADM requires delegated (f410) address + let owner = if let Some(ref owner_str) = args.owner { + crate::require_fil_addr_from_str(owner_str)? + } else { + let eth_addr = EthAddress::new_secp256k1(&pub_key.serialize()) + .context("failed to derive delegated address")?; + Address::new_delegated(10, ð_addr.0) + .context("failed to construct f410 address")? + }; + + let state = fm_client + .actor_state(&addr, FvmQueryHeight::default()) + .await + .context("Failed to get actor state")?; + let sequence = state.value.map(|(_, s)| s.sequence).unwrap_or(0); + + let mf = SignedMessageFactory::new(secret_key, addr, sequence, ChainID::from(chain_id)); + let mut bound_client = fm_client.bind(mf); + + // Parse metadata + let metadata = parse_metadata(&args.metadata)?; + + let params = CreateExternalParams { + owner, + kind: Kind::Bucket, + metadata, + }; + + let params_bytes = + RawBytes::serialize(params).context("Failed to serialize CreateExternalParams")?; + + let gas_params = GasParams { + gas_limit: 10_000_000_000, + gas_fee_cap: TokenAmount::from_atto(10_000), + gas_premium: TokenAmount::from_atto(10_000), + }; + + println!("Creating bucket..."); + + let res = TxClient::::transaction( + &mut bound_client, + adm::ADM_ACTOR_ADDR, + AdmMethod::CreateExternal as u64, + params_bytes, + TokenAmount::zero(), + gas_params, + ) + .await + .context("Failed to send CreateExternal transaction")?; + + if res.response.check_tx.code.is_err() { + return Err(anyhow!( + "CreateExternal check_tx failed: {}", + res.response.check_tx.log + )); + } + + if res.response.deliver_tx.code.is_err() { + let log = &res.response.deliver_tx.log; + let info = &res.response.deliver_tx.info; + return Err(anyhow!( + "CreateExternal deliver_tx failed (code {:?}): log={} info={}", + res.response.deliver_tx.code, + if log.is_empty() { "" } else { log }, + if info.is_empty() { "" } else { info }, + )); + } + + // Decode the return value + let return_data = fendermint_rpc::response::decode_data(&res.response.deliver_tx.data) + .context("Failed to decode response data")?; + let result: CreateExternalReturn = fvm_ipld_encoding::from_slice(&return_data) + .context("Failed to decode CreateExternalReturn")?; + + let bucket_id_addr = Address::new_id(result.actor_id); + let evm_addr = fendermint_vm_actor_interface::eam::EthAddress::from_id(result.actor_id); + + println!("Bucket created successfully!"); + println!(" Actor ID: {}", bucket_id_addr); + println!(" EVM address: 0x{}", hex::encode(evm_addr.0)); + if let Some(ref robust) = result.robust_address { + println!(" Robust addr: {}", robust); + } + println!(" Owner: {}", owner); + + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// List +// --------------------------------------------------------------------------- + +#[derive(Debug, Args)] +pub struct ListBucketsArgs { + /// Storage config file + #[arg(long)] + pub config: Option, + + /// Owner address to list buckets for (defaults to operator key address) + #[arg(long)] + pub owner: Option, + + /// Output in JSON format + #[arg(long)] + pub json: bool, +} + +pub struct ListBuckets; + +#[async_trait] +impl CommandLineHandler for ListBuckets { + type Arguments = ListBucketsArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + let fm_client = FendermintClient::new_http(config.tendermint_rpc_url.parse()?, None)?; + + // Determine the owner address + let owner = if let Some(ref owner_str) = args.owner { + crate::require_fil_addr_from_str(owner_str)? + } else { + let secret_key = + SignedMessageFactory::read_secret_key(&config.secret_key_file)?; + Address::new_secp256k1(&secret_key.public_key().serialize())? + }; + + let params = ListMetadataParams { owner }; + let params_bytes = + RawBytes::serialize(params).context("Failed to serialize ListMetadataParams")?; + + let msg = fvm_shared::message::Message { + version: Default::default(), + from: fendermint_vm_actor_interface::system::SYSTEM_ACTOR_ADDR, + to: adm::ADM_ACTOR_ADDR, + sequence: 0, + value: TokenAmount::zero(), + method_num: AdmMethod::ListMetadata as u64, + params: params_bytes, + gas_limit: 10_000_000_000, + gas_fee_cap: TokenAmount::zero(), + gas_premium: TokenAmount::zero(), + }; + + let response = fm_client + .call(msg, FvmQueryHeight::default()) + .await + .context("Failed to query ListMetadata")?; + + if response.value.code.is_err() { + return Err(anyhow!( + "ListMetadata query failed: {}", + response.value.info + )); + } + + let return_data = fendermint_rpc::response::decode_data(&response.value.data) + .context("Failed to decode response data")?; + + let results: Vec = fvm_ipld_encoding::from_slice(&return_data) + .context("Failed to decode ListMetadata response")?; + + if args.json { + let json_items: Vec = results + .iter() + .map(|m| { + serde_json::json!({ + "kind": format!("{}", m.kind), + "address": m.address.to_string(), + "metadata": m.metadata, + }) + }) + .collect(); + println!("{}", serde_json::to_string_pretty(&json_items)?); + } else { + let buckets: Vec<&adm::Metadata> = results + .iter() + .filter(|m| matches!(m.kind, Kind::Bucket)) + .collect(); + + if buckets.is_empty() { + println!("No buckets found for {}", owner); + } else { + println!("{:<50} METADATA", "ADDRESS"); + println!("{}", "-".repeat(80)); + for m in &buckets { + let meta_str = if m.metadata.is_empty() { + String::from("-") + } else { + m.metadata + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect::>() + .join(", ") + }; + println!("{:<50} {}", m.address, meta_str); + } + println!("\nTotal: {} buckets", buckets.len()); + } + } + + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn parse_metadata(pairs: &[String]) -> Result> { + let mut map = HashMap::new(); + for pair in pairs { + let (key, value) = pair + .split_once('=') + .ok_or_else(|| anyhow!("Invalid metadata format '{}', expected KEY=VALUE", pair))?; + map.insert(key.to_string(), value.to_string()); + } + Ok(map) +} diff --git a/ipc/cli/src/commands/storage/cat.rs b/ipc/cli/src/commands/storage/cat.rs new file mode 100644 index 0000000000..66e83b7d9b --- /dev/null +++ b/ipc/cli/src/commands/storage/cat.rs @@ -0,0 +1,81 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Cat command for displaying file contents from storage + +use anyhow::{anyhow, Context, Result}; +use clap::Args; +use std::io::{self, Write}; +use std::path::PathBuf; + +use async_trait::async_trait; + +use crate::commands::storage::{client::GatewayClient, config::StorageConfig, path}; +use crate::{CommandLineHandler, GlobalArguments}; + +#[derive(Debug, Args)] +pub struct CatArgs { + /// Storage path (ipc://bucket_address/path/to/file) + #[arg(value_name = "PATH")] + pub path: String, + + /// Gateway URL (overrides config and env var) + #[arg(long)] + pub gateway: Option, + + /// Storage config file + #[arg(long)] + pub config: Option, +} + +pub struct CatStorage; + +#[async_trait] +impl CommandLineHandler for CatStorage { + type Arguments = CatArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let storage_path = path::StoragePath::parse(&args.path)?; + + if storage_path.is_bucket_root() { + return Err(anyhow!("Path must include a file key, not just bucket address")); + } + + // Load config + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let mut config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + // Get gateway URL + let gateway_url = config.get_gateway_url( + args.gateway.as_deref(), + Some(&config_path), + false, // not interactive (avoid prompting when piping) + )?; + + // Download from gateway + let gateway_client = GatewayClient::new(gateway_url)?; + let data = gateway_client + .download_object(&storage_path.bucket_address, &storage_path.key, None) + .await + .context("Failed to download object from gateway")?; + + // Write to stdout + io::stdout().write_all(&data)?; + io::stdout().flush()?; + + Ok(()) + } +} diff --git a/ipc/cli/src/commands/storage/client.rs b/ipc/cli/src/commands/storage/client.rs new file mode 100644 index 0000000000..88a88e932c --- /dev/null +++ b/ipc/cli/src/commands/storage/client.rs @@ -0,0 +1,241 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! HTTP client for storage gateway API +//! +//! This module provides functions to interact with the storage gateway's HTTP API. + +use anyhow::{anyhow, Context, Result}; +use fvm_shared::address::Address; +use reqwest::multipart::{Form, Part}; +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +/// Response from uploading a blob to the gateway +#[derive(Debug, Serialize, Deserialize)] +pub struct UploadResponse { + /// Original blob content hash + pub hash: String, + /// Number of chunks the data was split into + pub num_chunks: usize, + /// Number of data shards per chunk (k) + pub data_shards: usize, + /// Number of parity shards per chunk (m) + pub parity_shards: usize, + /// Original data length in bytes + pub original_len: usize, +} + +/// Node address information from the gateway +#[derive(Debug, Serialize, Deserialize)] +pub struct NodeInfo { + pub node_id: String, + pub relay_url: Option, + pub direct_addresses: Vec, +} + +/// Storage gateway HTTP client +pub struct GatewayClient { + base_url: String, + client: reqwest::Client, +} + +impl GatewayClient { + /// Create a new gateway client + pub fn new(base_url: String) -> Result { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(300)) // 5 minute timeout for large uploads + .build() + .context("Failed to create HTTP client")?; + + Ok(Self { base_url, client }) + } + + /// Upload a blob to the gateway + /// + /// This uploads the data to Iroh, erasure-encodes it, and distributes shards + /// to the assigned storage nodes. + pub async fn upload_blob(&self, data: Vec) -> Result { + let size = data.len() as u64; + + // Build multipart form + let form = Form::new() + .text("size", size.to_string()) + .part( + "data", + Part::bytes(data) + .file_name("upload") + .mime_str("application/octet-stream")?, + ); + + let url = format!("{}/v1/objects", self.base_url); + + let response = self + .client + .post(&url) + .multipart(form) + .send() + .await + .context("Failed to send upload request")?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow!( + "Upload failed with status {}: {}", + status, + body + )); + } + + let upload_response: UploadResponse = response + .json() + .await + .context("Failed to parse upload response")?; + + Ok(upload_response) + } + + /// Download a blob directly by its hash + /// + /// This retrieves the blob from storage nodes, RS-decodes it, and returns + /// the plaintext data. + pub async fn download_blob(&self, blob_hash: &str, height: Option) -> Result> { + // Remove 0x prefix if present + let hash_hex = blob_hash.strip_prefix("0x").unwrap_or(blob_hash); + + let mut url = format!("{}/v1/blobs/{}", self.base_url, hash_hex); + if let Some(h) = height { + url.push_str(&format!("?height={}", h)); + } + + let response = self + .client + .get(&url) + .send() + .await + .context("Failed to send download request")?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow!( + "Download failed with status {}: {}", + status, + body + )); + } + + let data = response + .bytes() + .await + .context("Failed to read download response")?; + + Ok(data.to_vec()) + } + + /// Download an object from a bucket by its key/path + /// + /// This queries the bucket contract for the object metadata, then retrieves + /// the blob data. + pub async fn download_object( + &self, + bucket_address: &Address, + key: &str, + height: Option, + ) -> Result> { + let bucket_str = bucket_address.to_string(); + let encoded_key = urlencoding::encode(key); + + let mut url = format!("{}/v1/objects/{}/{}", self.base_url, bucket_str, encoded_key); + if let Some(h) = height { + url.push_str(&format!("?height={}", h)); + } + + let response = self + .client + .get(&url) + .send() + .await + .context("Failed to send download request")?; + + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Err(anyhow!("Object not found: {}", key)); + } + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow!( + "Download failed with status {}: {}", + status, + body + )); + } + + let data = response + .bytes() + .await + .context("Failed to read download response")?; + + Ok(data.to_vec()) + } + + /// Get the gateway node information + pub async fn get_node_info(&self) -> Result { + let url = format!("{}/v1/node", self.base_url); + + let response = self + .client + .get(&url) + .send() + .await + .context("Failed to send node info request")?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow!( + "Node info request failed with status {}: {}", + status, + body + )); + } + + let node_info: NodeInfo = response + .json() + .await + .context("Failed to parse node info response")?; + + Ok(node_info) + } + + /// Check if the gateway is healthy and reachable + pub async fn health_check(&self) -> Result<()> { + let url = format!("{}/health", self.base_url); + + let response = self + .client + .get(&url) + .send() + .await + .context("Failed to send health check request")?; + + if !response.status().is_success() { + return Err(anyhow!("Gateway health check failed")); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_client_creation() { + let client = GatewayClient::new("http://localhost:8080".to_string()); + assert!(client.is_ok()); + } +} diff --git a/ipc/cli/src/commands/storage/config.rs b/ipc/cli/src/commands/storage/config.rs new file mode 100644 index 0000000000..1279e82854 --- /dev/null +++ b/ipc/cli/src/commands/storage/config.rs @@ -0,0 +1,146 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +use anyhow::Result; +use fs_err as fs; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; + +/// Which storage components to run. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum StorageRunMode { + Node, + Gateway, + #[default] + Both, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct StorageConfig { + /// Storage home directory for keys, iroh data, etc. + pub node_home: PathBuf, + /// Path to ipc-storage node binary (ipc-decentralized-storage `node`). + pub storage_node_bin: PathBuf, + /// Path to ipc-storage gateway binary. + pub storage_gateway_bin: PathBuf, + + /// FM network passed to storage binaries (testnet/mainnet). + pub network: String, + + /// Tendermint RPC endpoint of the subnet node. + pub tendermint_rpc_url: String, + /// EVM JSON-RPC endpoint of the subnet node. + pub eth_rpc_url: String, + + /// Secp256k1 key for signing chain transactions. + pub secret_key_file: PathBuf, + /// BLS key used by storage node/operator. + pub bls_key_file: PathBuf, + + /// Operator API URL published on-chain during registration. + pub operator_rpc_url: String, + + /// Run mode for `ipc-cli storage run`. + pub run_mode: StorageRunMode, + + /// Storage-node settings + pub node_rpc_bind_addr: String, + pub iroh_node_path: PathBuf, + pub iroh_node_v4_addr: Option, + pub node_batch_size: u32, + pub node_poll_interval_secs: u64, + pub node_max_concurrent_downloads: usize, + + /// Gateway settings + pub objects_listen_addr: String, + pub iroh_gateway_path: PathBuf, + pub iroh_gateway_v4_addr: Option, + + /// Gateway URL for storage CLI operations (optional) + /// If not set, CLI commands will check env var or prompt + pub gateway_url: Option, +} + +impl StorageConfig { + pub fn load>(path: P) -> Result { + let path = path.as_ref(); + let contents = fs::read_to_string(path)?; + let cfg: StorageConfig = serde_yaml::from_str(&contents) + .map_err(|e| anyhow::anyhow!("Failed to parse {}: {}", path.display(), e))?; + Ok(cfg) + } + + pub fn save>(&self, path: P) -> Result<()> { + let path = path.as_ref(); + let contents = serde_yaml::to_string(self)?; + fs::write(path, contents)?; + Ok(()) + } + + /// Get the gateway URL from various sources + /// + /// Priority order: + /// 1. Explicit gateway_url parameter (CLI flag) + /// 2. IPC_STORAGE_GATEWAY environment variable + /// 3. gateway_url field in config file + /// 4. Prompt user if interactive is true + /// + /// If a new URL is discovered via prompt, it will be saved to the config. + pub fn get_gateway_url>( + &mut self, + explicit_url: Option<&str>, + config_path: Option

, + interactive: bool, + ) -> Result { + // 1. Check explicit URL (CLI flag) + if let Some(url) = explicit_url { + return Ok(url.to_string()); + } + + // 2. Check environment variable + if let Ok(url) = std::env::var("IPC_STORAGE_GATEWAY") { + if !url.is_empty() { + return Ok(url); + } + } + + // 3. Check config file + if let Some(url) = &self.gateway_url { + if !url.is_empty() { + return Ok(url.clone()); + } + } + + // 4. Prompt user if interactive + if interactive { + println!("Gateway URL not configured."); + println!("Please enter the storage gateway URL (e.g., http://localhost:8080):"); + + let mut input = String::new(); + std::io::stdin().read_line(&mut input)?; + let url = input.trim().to_string(); + + if url.is_empty() { + anyhow::bail!("Gateway URL cannot be empty"); + } + + // Save to config if path provided + if let Some(path) = config_path { + self.gateway_url = Some(url.clone()); + self.save(path)?; + println!("Gateway URL saved to config."); + } + + return Ok(url); + } + + anyhow::bail!( + "Gateway URL not configured. Set via:\n\ + 1. --gateway flag\n\ + 2. IPC_STORAGE_GATEWAY environment variable\n\ + 3. gateway_url in storage config file" + ) + } +} diff --git a/ipc/cli/src/commands/storage/cp.rs b/ipc/cli/src/commands/storage/cp.rs new file mode 100644 index 0000000000..0649d3c31b --- /dev/null +++ b/ipc/cli/src/commands/storage/cp.rs @@ -0,0 +1,475 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Copy command for storage operations +//! +//! Supports three modes: +//! - local -> ipc:// : Upload to storage +//! - ipc:// -> local : Download from storage +//! - ipc:// -> ipc:// : Copy between buckets + +use anyhow::{anyhow, Context, Result}; +use clap::Args; +use fs_err as fs; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; +use fendermint_rpc::client::FendermintClient; +use fendermint_rpc::message::SignedMessageFactory; +use fendermint_rpc::QueryClient; +use fendermint_vm_actor_interface::eam::EthAddress; +use fvm_shared::address::Address; +use fvm_shared::chainid::ChainID; + +use crate::commands::storage::{bucket, client::GatewayClient, config::StorageConfig, path}; +use crate::{CommandLineHandler, GlobalArguments}; + +#[derive(Debug, Args)] +pub struct CopyArgs { + /// Source path (local or ipc://) + #[arg(value_name = "SOURCE")] + pub source: String, + + /// Destination path (local or ipc://) + #[arg(value_name = "DEST")] + pub dest: String, + + /// Gateway URL (overrides config and env var) + #[arg(long)] + pub gateway: Option, + + /// Storage config file + #[arg(long)] + pub config: Option, + + /// Recursive copy (for directories) + #[arg(short, long)] + pub recursive: bool, + + /// Overwrite existing objects + #[arg(long)] + pub overwrite: bool, +} + +pub struct CopyStorage; + +#[async_trait] +impl CommandLineHandler for CopyStorage { + type Arguments = CopyArgs; + + async fn handle(global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let source_is_storage = path::is_storage_path(&args.source); + let dest_is_storage = path::is_storage_path(&args.dest); + + match (source_is_storage, dest_is_storage) { + (false, true) => { + // Local -> Storage (upload) + upload_to_storage(global, args).await + } + (true, false) => { + // Storage -> Local (download) + download_from_storage(global, args).await + } + (true, true) => { + // Storage -> Storage (copy between buckets) + copy_between_buckets(global, args).await + } + (false, false) => { + Err(anyhow!( + "At least one path must be a storage path (ipc://...)" + )) + } + } + } +} + +/// Upload a local file to storage +async fn upload_to_storage(_global: &GlobalArguments, args: &CopyArgs) -> Result<()> { + let local_path = Path::new(&args.source); + let storage_path = path::StoragePath::parse(&args.dest)?; + + if storage_path.is_bucket_root() { + return Err(anyhow!( + "Destination must include a key/path, not just bucket address" + )); + } + + // Handle recursive directory upload + if local_path.is_dir() { + if !args.recursive { + return Err(anyhow!( + "Cannot copy directory without -r/--recursive flag" + )); + } + return upload_directory(local_path, &storage_path, args).await; + } + + // Upload single file + upload_file(local_path, &storage_path, args).await +} + +/// Upload a single file to storage +async fn upload_file( + local_path: &Path, + storage_path: &path::StoragePath, + args: &CopyArgs, +) -> Result<()> { + println!("Uploading {} -> {}", local_path.display(), storage_path.to_uri()); + + // Read file data + let data = fs::read(local_path) + .with_context(|| format!("Failed to read file: {}", local_path.display()))?; + + // Get/load config + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let mut config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + // Get gateway URL + let gateway_url = config.get_gateway_url( + args.gateway.as_deref(), + Some(&config_path), + true, // interactive + )?; + + // Upload to gateway + let gateway_client = GatewayClient::new(gateway_url)?; + let upload_response = gateway_client + .upload_blob(data) + .await + .context("Failed to upload blob to gateway")?; + + println!( + "Uploaded blob: {} ({} bytes, {} chunks)", + upload_response.hash, upload_response.original_len, upload_response.num_chunks + ); + + // Convert hash to B256 (auto-detects hex or base32) + let blob_hash = bucket::hash_to_b256(&upload_response.hash) + .context("Invalid blob hash from gateway")?; + + // Get node info for source ID (auto-detects hex or base32) + let node_info = gateway_client.get_node_info().await?; + let source = bucket::hash_to_b256(&node_info.node_id) + .context("Invalid node ID from gateway")?; + + // Register object on-chain + println!("Registering object on-chain..."); + + // Create FendermintClient for on-chain operations + let fm_client = FendermintClient::new_http( + config.tendermint_rpc_url.parse()?, + None, + )?; + + // Query chain ID from the network + let chain_id = bucket::query_chain_id(&fm_client) + .await + .context("Failed to query chain ID")?; + + // Create bound client with secret key + let secret_key = SignedMessageFactory::read_secret_key( + &config.secret_key_file + )?; + + // Use delegated (f410) address as sender + let pub_key = secret_key.public_key(); + let eth_addr = EthAddress::new_secp256k1(&pub_key.serialize()) + .context("failed to derive delegated address")?; + let addr = Address::new_delegated(10, ð_addr.0) + .context("failed to construct f410 address")?; + let state = fm_client + .actor_state(&addr, fendermint_vm_message::query::FvmQueryHeight::default()) + .await + .context("Failed to get actor state")?; + let sequence = state.value.map(|(_, s)| s.sequence).unwrap_or(0); + + let mf = SignedMessageFactory::new(secret_key, addr, sequence, ChainID::from(chain_id)); + let mut bound_client = fm_client.bind(mf); + + // Add object to bucket + bucket::add_object( + &mut bound_client, + storage_path.bucket_address, + source, + storage_path.key.clone(), + blob_hash, + blob_hash, // recovery_hash (same as blob hash for now) + upload_response.original_len as u64, + HashMap::new(), // metadata + upload_response.data_shards as u16, + upload_response.parity_shards as u16, + ) + .await + .map_err(|e| anyhow::anyhow!("Failed to register object on-chain: {:?}", e))?; + + println!("✓ Successfully uploaded and registered: {}", storage_path.key); + + Ok(()) +} + +/// Upload a directory recursively +async fn upload_directory( + local_dir: &Path, + storage_base: &path::StoragePath, + args: &CopyArgs, +) -> Result<()> { + println!("Uploading directory {} recursively...", local_dir.display()); + + // Walk directory and upload each file + for entry in walkdir::WalkDir::new(local_dir) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + { + if entry.file_type().is_file() { + let rel_path = entry.path().strip_prefix(local_dir)?; + let rel_path_str = rel_path.to_string_lossy().to_string(); + + // Construct storage key + let storage_key = if storage_base.key.is_empty() { + rel_path_str + } else { + format!("{}/{}", storage_base.key.trim_end_matches('/'), rel_path_str) + }; + + let file_storage_path = path::StoragePath { + bucket_address: storage_base.bucket_address, + key: storage_key, + }; + + // Upload file + upload_file(entry.path(), &file_storage_path, args).await?; + } + } + + println!("✓ Directory upload complete"); + Ok(()) +} + +/// Download a file from storage to local +async fn download_from_storage(_global: &GlobalArguments, args: &CopyArgs) -> Result<()> { + let storage_path = path::StoragePath::parse(&args.source)?; + let local_path = Path::new(&args.dest); + + // Handle recursive directory download + if storage_path.is_bucket_root() || storage_path.key.ends_with('/') { + if !args.recursive { + return Err(anyhow!( + "Cannot download directory without -r/--recursive flag" + )); + } + return download_directory(&storage_path, local_path, args).await; + } + + // Download single file + download_file(&storage_path, local_path, args).await +} + +/// Download a single file from storage +async fn download_file( + storage_path: &path::StoragePath, + local_path: &Path, + args: &CopyArgs, +) -> Result<()> { + println!("Downloading {} -> {}", storage_path.to_uri(), local_path.display()); + + // Get/load config + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let mut config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + // Get gateway URL + let gateway_url = config.get_gateway_url( + args.gateway.as_deref(), + Some(&config_path), + true, // interactive + )?; + + // Download from gateway + let gateway_client = GatewayClient::new(gateway_url)?; + let data = gateway_client + .download_object(&storage_path.bucket_address, &storage_path.key, None) + .await + .context("Failed to download object from gateway")?; + + // Create parent directories if needed + if let Some(parent) = local_path.parent() { + fs::create_dir_all(parent)?; + } + + // Write to file + fs::write(local_path, data)?; + + println!("✓ Downloaded {} bytes", fs::metadata(local_path)?.len()); + + Ok(()) +} + +/// Download a directory recursively (list objects with prefix) +async fn download_directory( + storage_base: &path::StoragePath, + local_dir: &Path, + args: &CopyArgs, +) -> Result<()> { + println!("Downloading directory {} recursively...", storage_base.to_uri()); + + // Load config + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let mut config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + let gateway_url = config.get_gateway_url( + args.gateway.as_deref(), + Some(&config_path), + true, + )?; + + let gateway_client = GatewayClient::new(gateway_url)?; + + // List all objects with the prefix + let fm_client = FendermintClient::new_http( + config.tendermint_rpc_url.parse()?, + None, + )?; + + let prefix = if storage_base.is_bucket_root() { + None + } else { + Some(storage_base.key.clone()) + }; + + let mut start_key = None; + let mut downloaded_count = 0; + + loop { + let list_result = bucket::list_objects( + &fm_client, + storage_base.bucket_address, + prefix.clone(), + None, // no delimiter for recursive listing + start_key, + 100, + ) + .await + .context("Failed to list objects")?; + + if list_result.objects.is_empty() { + break; + } + + for (key_bytes, _) in &list_result.objects { + let key_str = String::from_utf8_lossy(key_bytes).to_string(); + + // Compute relative path by stripping the prefix + let rel_path = if let Some(ref pfx) = prefix { + key_str.strip_prefix(pfx).unwrap_or(&key_str) + } else { + &key_str + }; + let rel_path = rel_path.trim_start_matches('/'); + + let dest_file = local_dir.join(rel_path); + + // Create parent directories + if let Some(parent) = dest_file.parent() { + fs::create_dir_all(parent)?; + } + + // Download the object + println!("Downloading {} -> {}", key_str, dest_file.display()); + let data = gateway_client + .download_object(&storage_base.bucket_address, &key_str, None) + .await + .with_context(|| format!("Failed to download object: {}", key_str))?; + + fs::write(&dest_file, data)?; + downloaded_count += 1; + } + + if list_result.next_key.is_none() { + break; + } + + start_key = list_result.next_key.map(|k| String::from_utf8_lossy(&k).to_string()); + } + + println!("✓ Downloaded {} files", downloaded_count); + Ok(()) +} + +/// Copy an object between storage buckets +async fn copy_between_buckets(_global: &GlobalArguments, args: &CopyArgs) -> Result<()> { + let source_path = path::StoragePath::parse(&args.source)?; + let dest_path = path::StoragePath::parse(&args.dest)?; + + println!("Copying {} -> {}", source_path.to_uri(), dest_path.to_uri()); + + // Download from source + let temp_dir = tempfile::tempdir()?; + let temp_file = temp_dir.path().join("temp_copy"); + + let download_args = CopyArgs { + source: args.source.clone(), + dest: temp_file.to_string_lossy().to_string(), + gateway: args.gateway.clone(), + config: args.config.clone(), + recursive: false, + overwrite: args.overwrite, + }; + + download_file(&source_path, &temp_file, &download_args).await?; + + // Upload to destination + let upload_args = CopyArgs { + source: temp_file.to_string_lossy().to_string(), + dest: args.dest.clone(), + gateway: args.gateway.clone(), + config: args.config.clone(), + recursive: false, + overwrite: args.overwrite, + }; + + upload_file(&temp_file, &dest_path, &upload_args).await?; + + println!("✓ Copy complete"); + + Ok(()) +} diff --git a/ipc/cli/src/commands/storage/credit.rs b/ipc/cli/src/commands/storage/credit.rs new file mode 100644 index 0000000000..f5a9319d66 --- /dev/null +++ b/ipc/cli/src/commands/storage/credit.rs @@ -0,0 +1,310 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Credit subcommand for buying and querying storage credits. + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use clap::{Args, Subcommand}; +use num_traits::Zero; +use std::path::PathBuf; + +use fendermint_actor_blobs_shared::{ + accounts::Account, + credit::BuyCreditParams, + method::Method as BlobsMethod, + BLOBS_ACTOR_ADDR, +}; +use fendermint_rpc::client::FendermintClient; +use fendermint_rpc::message::{GasParams, SignedMessageFactory}; +use fendermint_rpc::tx::{TxClient, TxCommit}; +use fendermint_rpc::QueryClient; +use fendermint_vm_message::query::FvmQueryHeight; +use fvm_ipld_encoding::RawBytes; +use fendermint_vm_actor_interface::eam::EthAddress; +use fvm_shared::address::Address; +use fvm_shared::chainid::ChainID; +use fvm_shared::econ::TokenAmount; + +use crate::commands::storage::bucket; +use crate::commands::storage::config::StorageConfig; +use crate::{CommandLineHandler, GlobalArguments}; + +#[derive(Debug, Args)] +#[command(name = "credit", about = "Buy and query storage credits")] +pub struct CreditCommandArgs { + #[command(subcommand)] + command: CreditCommands, +} + +#[derive(Debug, Subcommand)] +pub enum CreditCommands { + /// Buy storage credits by sending tokens to the blobs actor + Buy(BuyCreditArgs), + /// Get account credit information + Info(CreditInfoArgs), +} + +impl CreditCommandArgs { + pub async fn handle(&self, global: &GlobalArguments) -> anyhow::Result<()> { + match &self.command { + CreditCommands::Buy(args) => BuyCredit::handle(global, args).await, + CreditCommands::Info(args) => CreditInfo::handle(global, args).await, + } + } +} + +// --------------------------------------------------------------------------- +// Buy +// --------------------------------------------------------------------------- + +#[derive(Debug, Args)] +pub struct BuyCreditArgs { + /// Amount of tokens to spend (in FIL/ether units, e.g. 0.1) + #[arg(value_name = "AMOUNT")] + pub amount: f64, + + /// Recipient address (defaults to the operator key address) + #[arg(long)] + pub to: Option, + + /// Storage config file + #[arg(long)] + pub config: Option, +} + +pub struct BuyCredit; + +#[async_trait] +impl CommandLineHandler for BuyCredit { + type Arguments = BuyCreditArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + if args.amount <= 0.0 { + return Err(anyhow!("Amount must be positive")); + } + + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + let fm_client = FendermintClient::new_http(config.tendermint_rpc_url.parse()?, None)?; + + let chain_id = bucket::query_chain_id(&fm_client) + .await + .context("Failed to query chain ID")?; + + let secret_key = SignedMessageFactory::read_secret_key(&config.secret_key_file)?; + let pub_key = secret_key.public_key(); + let addr = Address::new_secp256k1(&pub_key.serialize())?; + + // Determine recipient — blobs actor requires a delegated (f410) address + let recipient = if let Some(ref to_str) = args.to { + crate::require_fil_addr_from_str(to_str)? + } else { + let eth_addr = EthAddress::new_secp256k1(&pub_key.serialize()) + .context("failed to derive delegated address from operator key")?; + Address::new_delegated(10, ð_addr.0) + .context("failed to construct f410 address")? + }; + + let state = fm_client + .actor_state(&addr, FvmQueryHeight::default()) + .await + .context("Failed to get actor state")?; + let sequence = state.value.map(|(_, s)| s.sequence).unwrap_or(0); + + let mf = SignedMessageFactory::new(secret_key, addr, sequence, ChainID::from(chain_id)); + let mut bound_client = fm_client.bind(mf); + + let params = BuyCreditParams(recipient); + let params_bytes = + RawBytes::serialize(params).context("Failed to serialize BuyCreditParams")?; + + // Convert amount to TokenAmount (nano precision) + let value = crate::f64_to_token_amount(args.amount)?; + + let gas_params = GasParams { + gas_limit: 10_000_000_000, + gas_fee_cap: TokenAmount::from_atto(10_000), + gas_premium: TokenAmount::from_atto(10_000), + }; + + println!("Buying credit for {} with {} FIL...", recipient, args.amount); + + let res = TxClient::::transaction( + &mut bound_client, + BLOBS_ACTOR_ADDR, + BlobsMethod::BuyCredit as u64, + params_bytes, + value, + gas_params, + ) + .await + .context("Failed to send BuyCredit transaction")?; + + if res.response.check_tx.code.is_err() { + return Err(anyhow!( + "BuyCredit check_tx failed: {}", + res.response.check_tx.log + )); + } + + if res.response.deliver_tx.code.is_err() { + let log = &res.response.deliver_tx.log; + let info = &res.response.deliver_tx.info; + return Err(anyhow!( + "BuyCredit deliver_tx failed (code {:?}): log={} info={}", + res.response.deliver_tx.code, + if log.is_empty() { "" } else { log }, + if info.is_empty() { "" } else { info }, + )); + } + + println!("Credit purchased successfully for {}", recipient); + + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Info +// --------------------------------------------------------------------------- + +#[derive(Debug, Args)] +pub struct CreditInfoArgs { + /// Account address to query (defaults to operator key address) + #[arg(long)] + pub address: Option, + + /// Storage config file + #[arg(long)] + pub config: Option, + + /// Output in JSON format + #[arg(long)] + pub json: bool, +} + +pub struct CreditInfo; + +#[async_trait] +impl CommandLineHandler for CreditInfo { + type Arguments = CreditInfoArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + let fm_client = FendermintClient::new_http(config.tendermint_rpc_url.parse()?, None)?; + + // Determine the address to query + let query_addr = if let Some(ref addr_str) = args.address { + crate::require_fil_addr_from_str(addr_str)? + } else { + let secret_key = SignedMessageFactory::read_secret_key(&config.secret_key_file)?; + Address::new_secp256k1(&secret_key.public_key().serialize())? + }; + + // Query the GetAccount method on the blobs actor + let params_bytes = + RawBytes::serialize(query_addr).context("Failed to serialize address")?; + + let msg = fvm_shared::message::Message { + version: Default::default(), + from: fendermint_vm_actor_interface::system::SYSTEM_ACTOR_ADDR, + to: BLOBS_ACTOR_ADDR, + sequence: 0, + value: TokenAmount::zero(), + method_num: BlobsMethod::GetAccount as u64, + params: params_bytes, + gas_limit: 10_000_000_000, + gas_fee_cap: TokenAmount::zero(), + gas_premium: TokenAmount::zero(), + }; + + let response = fm_client + .call(msg, FvmQueryHeight::default()) + .await + .context("Failed to query GetAccount")?; + + if response.value.code.is_err() { + return Err(anyhow!( + "GetAccount query failed: {}", + response.value.info + )); + } + + let return_data = fendermint_rpc::response::decode_data(&response.value.data) + .context("Failed to decode response data")?; + + let account: Option = fvm_ipld_encoding::from_slice(&return_data) + .context("Failed to decode Account")?; + + match account { + Some(acct) => { + if args.json { + let output = serde_json::json!({ + "address": query_addr.to_string(), + "capacity_used": acct.capacity_used, + "credit_free": acct.credit_free.atto().to_string(), + "credit_committed": acct.credit_committed.atto().to_string(), + "credit_sponsor": acct.credit_sponsor.map(|a| a.to_string()), + "last_debit_epoch": acct.last_debit_epoch, + "max_ttl": acct.max_ttl, + "gas_allowance": acct.gas_allowance.atto().to_string(), + "approvals_to": acct.approvals_to.len(), + "approvals_from": acct.approvals_from.len(), + }); + println!("{}", serde_json::to_string_pretty(&output)?); + } else { + println!("Account: {}", query_addr); + println!(" Capacity used: {} bytes", acct.capacity_used); + println!(" Credit free: {}", acct.credit_free); + println!(" Credit committed: {}", acct.credit_committed); + if let Some(sponsor) = &acct.credit_sponsor { + println!(" Credit sponsor: {}", sponsor); + } + println!(" Last debit epoch: {}", acct.last_debit_epoch); + println!(" Max TTL: {} epochs", acct.max_ttl); + println!(" Gas allowance: {}", acct.gas_allowance); + println!( + " Approvals: {} outgoing, {} incoming", + acct.approvals_to.len(), + acct.approvals_from.len() + ); + } + } + None => { + println!("No account found for {}", query_addr); + } + } + + Ok(()) + } +} diff --git a/ipc/cli/src/commands/storage/init.rs b/ipc/cli/src/commands/storage/init.rs new file mode 100644 index 0000000000..86e2cf740d --- /dev/null +++ b/ipc/cli/src/commands/storage/init.rs @@ -0,0 +1,175 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +use crate::commands::storage::config::{StorageConfig, StorageRunMode}; +use crate::CommandLineHandler; +use anyhow::{Context, Result}; +use async_trait::async_trait; +use clap::Args; +use fendermint_crypto::{to_b64, SecretKey}; +use fendermint_vm_actor_interface::eam::EthAddress; +use fvm_shared::address::{set_current_network, Address, Network}; +use rand::thread_rng; +use std::fs; +use std::path::{Path, PathBuf}; + +pub(crate) struct InitStorage; + +#[async_trait] +impl CommandLineHandler for InitStorage { + type Arguments = InitStorageArgs; + + async fn handle(global: &crate::GlobalArguments, args: &Self::Arguments) -> Result<()> { + let node_home = args + .home + .clone() + .unwrap_or_else(|| default_storage_home()); + + let storage_dir = node_home.join("storage"); + + tokio::fs::create_dir_all(&storage_dir) + .await + .with_context(|| { + format!( + "failed to create storage directory at {}", + storage_dir.display() + ) + })?; + + let out = args + .out + .clone() + .unwrap_or_else(|| global.config_dir().join("storage.yaml")); + + let root = workspace_root(); + let secret_key_file = args + .secret_key_file + .clone() + .unwrap_or_else(|| storage_dir.join("operator.sk")); + ensure_operator_secret_key(&secret_key_file)?; + + // Storage tooling defaults to testnet addressing unless explicitly overridden at runtime. + set_current_network(Network::Testnet); + let operator_sk = read_operator_secret_key(&secret_key_file).with_context(|| { + format!( + "failed to read operator key from {}", + secret_key_file.display() + ) + })?; + let operator_pk = operator_sk.public_key(); + let operator_f1 = Address::new_secp256k1(&operator_pk.serialize()) + .context("failed to derive operator f1 address")?; + let operator_f410_eth = EthAddress::new_secp256k1(&operator_pk.serialize()) + .context("failed to derive operator delegated address")?; + let operator_f410 = Address::new_delegated(10, &operator_f410_eth.0) + .context("failed to construct operator f410 address")?; + let storage_cfg = StorageConfig { + node_home: node_home.clone(), + storage_node_bin: root.join("target/release/node"), + storage_gateway_bin: root.join("target/release/gateway"), + network: "testnet".to_string(), + tendermint_rpc_url: "http://127.0.0.1:26657".to_string(), + eth_rpc_url: "http://127.0.0.1:8545".to_string(), + secret_key_file, + bls_key_file: storage_dir.join("bls_key.hex"), + operator_rpc_url: "http://127.0.0.1:8081".to_string(), + run_mode: StorageRunMode::Both, + node_rpc_bind_addr: "127.0.0.1:8081".to_string(), + iroh_node_path: storage_dir.join("iroh-node"), + iroh_node_v4_addr: Some("0.0.0.0:11204".to_string()), + node_batch_size: 10, + node_poll_interval_secs: 5, + node_max_concurrent_downloads: 10, + objects_listen_addr: "127.0.0.1:8080".to_string(), + iroh_gateway_path: storage_dir.join("iroh-gateway"), + iroh_gateway_v4_addr: Some("0.0.0.0:11205".to_string()), + gateway_url: None, + }; + + storage_cfg + .save(&out) + .with_context(|| format!("failed to write storage config to {}", out.display()))?; + + log::info!("Storage configuration generated at: {}", out.display()); + log::info!( + "Optional: register operator with `ipc-cli storage run --config {} --register-operator`", + out.display() + ); + log::info!( + "Using operator secret key file: {}", + storage_cfg.secret_key_file.display() + ); + log::info!("Operator funding addresses:"); + log::info!(" - delegated (fund this): {}", operator_f410); + log::info!(" - native (diagnostic only): {}", operator_f1); + log::info!( + "Recommendation: cross-fund the delegated operator address above before `storage run --register-operator`." + ); + log::info!( + "Run storage services with `ipc-cli storage run --config {}`", + out.display() + ); + Ok(()) + } +} + +#[derive(Debug, Args)] +#[command(name = "init", about = "Generate default storage config")] +pub struct InitStorageArgs { + #[arg( + long, + help = "Storage home directory for keys and data (defaults to ~/.ipc-storage)" + )] + pub home: Option, + #[arg(long, help = "Output path for generated storage YAML config")] + pub out: Option, + #[arg( + long, + help = "Path to operator secp256k1 secret key file (base64). Defaults to /storage/operator.sk" + )] + pub secret_key_file: Option, +} + +fn default_storage_home() -> PathBuf { + dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".ipc-storage") +} + +fn workspace_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|p| p.parent()) + .unwrap_or_else(|| Path::new(".")) + .to_path_buf() +} + +fn ensure_operator_secret_key(path: &Path) -> Result<()> { + if path.exists() { + return Ok(()); + } + + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| { + format!( + "failed to create operator key directory at {}", + parent.display() + ) + })?; + } + + let mut rng = thread_rng(); + let sk = SecretKey::random(&mut rng); + let sk_bytes = sk.serialize(); + let sk_b64 = to_b64(sk_bytes.as_ref()); + fs::write(path, sk_b64) + .with_context(|| format!("failed to write operator key to {}", path.display()))?; + log::info!("Generated dedicated operator key at {}", path.display()); + Ok(()) +} + +fn read_operator_secret_key(path: &Path) -> Result { + let sk = fendermint_rpc::message::SignedMessageFactory::read_secret_key(path) + .with_context(|| format!("failed to parse secret key at {}", path.display()))?; + Ok(sk) +} diff --git a/ipc/cli/src/commands/storage/ls.rs b/ipc/cli/src/commands/storage/ls.rs new file mode 100644 index 0000000000..986d04a448 --- /dev/null +++ b/ipc/cli/src/commands/storage/ls.rs @@ -0,0 +1,217 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! List command for storage operations + +use anyhow::{anyhow, Context, Result}; +use clap::Args; +use std::path::PathBuf; + +use fendermint_rpc::client::FendermintClient; +use serde_json::json; + +use async_trait::async_trait; + +use crate::commands::storage::{bucket, config::StorageConfig, path}; +use crate::{CommandLineHandler, GlobalArguments}; + +#[derive(Debug, Args)] +pub struct ListArgs { + /// Storage path (ipc://bucket_address/prefix) + #[arg(value_name = "PATH")] + pub path: String, + + /// Storage config file + #[arg(long)] + pub config: Option, + + /// Output in JSON format + #[arg(long)] + pub json: bool, + + /// Show all details + #[arg(short, long)] + pub long: bool, + + /// Delimiter for hierarchical listing (default: none, S3-style: "/") + #[arg(short, long)] + pub delimiter: Option, + + /// Maximum number of objects to list + #[arg(long, default_value = "100")] + pub limit: u64, +} + +pub struct ListStorage; + +#[async_trait] +impl CommandLineHandler for ListStorage { + type Arguments = ListArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let storage_path = path::StoragePath::parse(&args.path)?; + + // Load config + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + // Create FendermintClient + let fm_client = FendermintClient::new_http( + config.tendermint_rpc_url.parse()?, + None, + )?; + + // List objects + let prefix = if storage_path.is_bucket_root() { + None + } else { + Some(storage_path.key.clone()) + }; + + let list_result = bucket::list_objects( + &fm_client, + storage_path.bucket_address, + prefix.clone(), + args.delimiter.clone(), + None, // start_key + args.limit, + ) + .await + .context("Failed to list objects")?; + + // Output results + if args.json { + print_json(&list_result, &prefix)?; + } else { + print_table(&list_result, &prefix, args.long)?; + } + + Ok(()) + } +} + +fn print_json( + result: &fendermint_actor_bucket::ListObjectsReturn, + _prefix: &Option, +) -> Result<()> { + let mut objects = Vec::new(); + for (key, state) in &result.objects { + let key_str = String::from_utf8_lossy(key); + objects.push(json!({ + "key": key_str, + "hash": format!("0x{}", hex::encode(state.hash.0)), + "size": state.size, + "expiry": state.expiry, + "metadata": state.metadata, + })); + } + + let mut prefixes = Vec::new(); + for prefix in &result.common_prefixes { + prefixes.push(String::from_utf8_lossy(prefix).to_string()); + } + + let output = json!({ + "objects": objects, + "common_prefixes": prefixes, + "next_key": result.next_key.as_ref().map(|k| String::from_utf8_lossy(k).to_string()), + }); + + println!("{}", serde_json::to_string_pretty(&output)?); + Ok(()) +} + +fn print_table( + result: &fendermint_actor_bucket::ListObjectsReturn, + _prefix: &Option, + long: bool, +) -> Result<()> { + if result.objects.is_empty() && result.common_prefixes.is_empty() { + println!("No objects found"); + return Ok(()); + } + + // Print common prefixes (directories) first + if !result.common_prefixes.is_empty() { + if long { + println!("{:<50} {:<10} {:<66}", "KEY", "SIZE", "HASH"); + println!("{}", "-".repeat(130)); + } + + for prefix_bytes in &result.common_prefixes { + let prefix_str = String::from_utf8_lossy(prefix_bytes); + if long { + println!("{:<50} {:<10} {:<66}", prefix_str, "DIR", "-"); + } else { + println!("{}", prefix_str); + } + } + } + + // Print objects + if !result.objects.is_empty() { + if long && result.common_prefixes.is_empty() { + println!("{:<50} {:<10} {:<66}", "KEY", "SIZE", "HASH"); + println!("{}", "-".repeat(130)); + } + + for (key, state) in &result.objects { + let key_str = String::from_utf8_lossy(key); + let hash_str = format!("0x{}", hex::encode(&state.hash.0[..8])); // Truncated hash + + if long { + println!( + "{:<50} {:<10} {:<66}", + key_str, + format_size(state.size), + hash_str + ); + } else { + println!("{}", key_str); + } + } + } + + // Print pagination info + if let Some(next_key) = &result.next_key { + let next_key_str = String::from_utf8_lossy(next_key); + println!("\n(More results available, next key: {})", next_key_str); + } + + println!( + "\nTotal: {} objects, {} prefixes", + result.objects.len(), + result.common_prefixes.len() + ); + + Ok(()) +} + +fn format_size(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + + if bytes >= GB { + format!("{:.1} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.1} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.1} KB", bytes as f64 / KB as f64) + } else { + format!("{} B", bytes) + } +} diff --git a/ipc/cli/src/commands/storage/mod.rs b/ipc/cli/src/commands/storage/mod.rs new file mode 100644 index 0000000000..dabb10c9e4 --- /dev/null +++ b/ipc/cli/src/commands/storage/mod.rs @@ -0,0 +1,84 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +pub mod bucket; +pub mod bucket_cmd; +pub mod cat; +pub mod credit; +pub mod client; +pub mod config; +pub mod cp; +pub mod init; +pub mod ls; +pub mod mv; +pub mod path; +pub mod rm; +pub mod run; +pub mod stat; +pub mod sync; + +use crate::commands::storage::bucket_cmd::BucketCommandArgs; +use crate::commands::storage::cat::{CatArgs, CatStorage}; +use crate::commands::storage::credit::CreditCommandArgs; +use crate::commands::storage::cp::{CopyArgs, CopyStorage}; +use crate::commands::storage::init::{InitStorage, InitStorageArgs}; +use crate::commands::storage::ls::{ListArgs, ListStorage}; +use crate::commands::storage::mv::{MoveArgs, MoveStorage}; +use crate::commands::storage::rm::{RemoveArgs, RemoveStorage}; +use crate::commands::storage::run::{RunStorage, RunStorageArgs}; +use crate::commands::storage::stat::{StatArgs, StatStorage}; +use crate::commands::storage::sync::{SyncArgs, SyncStorage}; +use crate::{CommandLineHandler, GlobalArguments}; +use clap::{Args, Subcommand}; + +#[derive(Debug, Args)] +#[command(name = "storage", about = "storage node automation and file operations")] +#[command(args_conflicts_with_subcommands = true)] +pub(crate) struct StorageCommandsArgs { + #[command(subcommand)] + command: Commands, +} + +impl StorageCommandsArgs { + pub async fn handle(&self, global: &GlobalArguments) -> anyhow::Result<()> { + match &self.command { + Commands::Bucket(args) => args.handle(global).await, + Commands::Credit(args) => args.handle(global).await, + Commands::Init(args) => InitStorage::handle(global, args).await, + Commands::Run(args) => RunStorage::handle(global, args).await, + Commands::Cp(args) => CopyStorage::handle(global, args).await, + Commands::Ls(args) => ListStorage::handle(global, args).await, + Commands::Cat(args) => CatStorage::handle(global, args).await, + Commands::Stat(args) => StatStorage::handle(global, args).await, + Commands::Rm(args) => RemoveStorage::handle(global, args).await, + Commands::Mv(args) => MoveStorage::handle(global, args).await, + Commands::Sync(args) => SyncStorage::handle(global, args).await, + } + } +} + +#[derive(Debug, Subcommand)] +pub(crate) enum Commands { + /// Create and manage storage buckets + Bucket(BucketCommandArgs), + /// Buy and query storage credits + Credit(CreditCommandArgs), + /// Initialize storage configuration + Init(InitStorageArgs), + /// Run storage node and/or gateway + Run(RunStorageArgs), + /// Copy files to/from storage + Cp(CopyArgs), + /// List objects in storage + Ls(ListArgs), + /// Display file contents from storage + Cat(CatArgs), + /// Show object metadata + Stat(StatArgs), + /// Remove objects from storage + Rm(RemoveArgs), + /// Move/rename objects in storage + Mv(MoveArgs), + /// Sync directories with storage + Sync(SyncArgs), +} diff --git a/ipc/cli/src/commands/storage/mv.rs b/ipc/cli/src/commands/storage/mv.rs new file mode 100644 index 0000000000..785c852346 --- /dev/null +++ b/ipc/cli/src/commands/storage/mv.rs @@ -0,0 +1,169 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Move/rename command for storage objects + +use anyhow::{anyhow, Context, Result}; +use clap::Args; +use std::path::PathBuf; + +use async_trait::async_trait; +use fendermint_rpc::client::FendermintClient; +use fendermint_rpc::message::SignedMessageFactory; +use fendermint_rpc::QueryClient; +use fendermint_vm_actor_interface::eam::EthAddress; +use fvm_shared::address::Address; +use fvm_shared::chainid::ChainID; + +use crate::commands::storage::{bucket, client::GatewayClient, config::StorageConfig, path}; +use crate::{CommandLineHandler, GlobalArguments}; + +#[derive(Debug, Args)] +pub struct MoveArgs { + /// Source storage path (ipc://bucket_address/path/to/file) + #[arg(value_name = "SOURCE")] + pub source: String, + + /// Destination storage path (ipc://bucket_address/path/to/newfile) + #[arg(value_name = "DEST")] + pub dest: String, + + /// Gateway URL (overrides config and env var) + #[arg(long)] + pub gateway: Option, + + /// Storage config file + #[arg(long)] + pub config: Option, +} + +pub struct MoveStorage; + +#[async_trait] +impl CommandLineHandler for MoveStorage { + type Arguments = MoveArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let source_path = path::StoragePath::parse(&args.source)?; + let dest_path = path::StoragePath::parse(&args.dest)?; + + if source_path.is_bucket_root() || dest_path.is_bucket_root() { + return Err(anyhow!("Paths must include file keys, not just bucket addresses")); + } + + println!("Moving {} -> {}", source_path.to_uri(), dest_path.to_uri()); + + // Load config + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let mut config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + // Create clients + let fm_client = FendermintClient::new_http( + config.tendermint_rpc_url.parse()?, + None, + )?; + + // Get source object metadata + let mut query_client = fm_client.clone(); + let source_object = bucket::get_object( + &mut query_client, + source_path.bucket_address, + source_path.key.clone(), + ) + .await + .context("Failed to get source object")?; + + let source_object = source_object.ok_or_else(|| anyhow!("Source object not found: {}", source_path.key))?; + + // Query chain ID from the network + let chain_id = bucket::query_chain_id(&fm_client) + .await + .context("Failed to query chain ID")?; + + // Create bound client for transactions + let secret_key = SignedMessageFactory::read_secret_key( + &config.secret_key_file + )?; + + let pub_key = secret_key.public_key(); + let eth_addr = EthAddress::new_secp256k1(&pub_key.serialize()) + .context("failed to derive delegated address")?; + let addr = Address::new_delegated(10, ð_addr.0) + .context("failed to construct f410 address")?; + let state = fm_client + .actor_state(&addr, fendermint_vm_message::query::FvmQueryHeight::default()) + .await + .context("Failed to get actor state")?; + let sequence = state.value.map(|(_, s)| s.sequence).unwrap_or(0); + + let mf = SignedMessageFactory::new(secret_key, addr, sequence, ChainID::from(chain_id)); + let mut bound_client = fm_client.bind(mf); + + // If moving within the same bucket, we can reuse the blob hash + if source_path.bucket_address == dest_path.bucket_address { + println!("Moving within same bucket (reusing blob)..."); + + // Get source node ID from gateway + let gateway_url = config.get_gateway_url( + args.gateway.as_deref(), + Some(&config_path), + true, + )?; + let gateway_client = GatewayClient::new(gateway_url)?; + let node_info = gateway_client.get_node_info().await?; + let source_node = bucket::hex_to_b256(&node_info.node_id) + .context("Invalid node ID from gateway")?; + + bucket::add_object( + &mut bound_client, + dest_path.bucket_address, + source_node, + dest_path.key.clone(), + source_object.hash, + source_object.recovery_hash, + source_object.size, + source_object.metadata.clone(), + 4, // data_shards - default + 2, // parity_shards - default + ) + .await + .context("Failed to add object at destination")?; + + println!("✓ Added at destination: {}", dest_path.key); + + // Delete from source location + bucket::delete_object( + &mut bound_client, + source_path.bucket_address, + source_path.key.clone(), + ) + .await + .context("Failed to delete source object")?; + + println!("✓ Deleted from source: {}", source_path.key); + } else { + // Moving across buckets requires re-upload + println!("Moving across buckets requires downloading and re-uploading..."); + return Err(anyhow!( + "Cross-bucket move not yet implemented. Use 'cp' followed by 'rm' instead." + )); + } + + println!("✓ Move complete"); + + Ok(()) + } +} diff --git a/ipc/cli/src/commands/storage/path.rs b/ipc/cli/src/commands/storage/path.rs new file mode 100644 index 0000000000..b858545c5b --- /dev/null +++ b/ipc/cli/src/commands/storage/path.rs @@ -0,0 +1,183 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Path parsing for storage URIs +//! +//! This module handles parsing of ipc:// URIs for storage operations. +//! Format: ipc://bucket_address/path/to/object +//! +//! Examples: +//! - ipc://0x1234.../documents/file.txt +//! - ipc://t1.../data/backup.tar.gz + +use anyhow::{anyhow, Context, Result}; +use fvm_shared::address::{Address, Error as NetworkError, Network}; +use std::str::FromStr; + +/// Represents a parsed storage path +#[derive(Debug, Clone)] +pub struct StoragePath { + /// The bucket contract address + pub bucket_address: Address, + /// The object key/path within the bucket + pub key: String, +} + +impl StoragePath { + /// Parse a storage URI in the format ipc://bucket_address/path/to/object + pub fn parse(uri: &str) -> Result { + if !uri.starts_with("ipc://") { + return Err(anyhow!("Storage path must start with ipc://")); + } + + let path_part = &uri[6..]; // Remove "ipc://" + + // Find the first '/' to separate bucket address from key + let (bucket_str, key) = match path_part.find('/') { + Some(idx) => { + let bucket = &path_part[..idx]; + let key = &path_part[idx + 1..]; // Skip the '/' + (bucket, key.to_string()) + } + None => { + // No key provided, just bucket address + (path_part, String::new()) + } + }; + + if bucket_str.is_empty() { + return Err(anyhow!("Bucket address cannot be empty")); + } + + let bucket_address = parse_address(bucket_str) + .with_context(|| format!("Invalid bucket address: {}", bucket_str))?; + + Ok(StoragePath { + bucket_address, + key, + }) + } + + /// Check if this path represents a bucket root (no key specified) + pub fn is_bucket_root(&self) -> bool { + self.key.is_empty() + } + + /// Get the parent directory path (for recursive operations) + pub fn parent(&self) -> Option { + if self.key.is_empty() { + return None; + } + + match self.key.rfind('/') { + Some(idx) => Some(self.key[..=idx].to_string()), + None => Some(String::new()), + } + } + + /// Check if this path is a prefix of another path (directory-like) + pub fn is_prefix_of(&self, other: &str) -> bool { + if self.key.is_empty() { + return true; + } + + other.starts_with(&self.key) && ( + other.len() == self.key.len() || + other.as_bytes().get(self.key.len()) == Some(&b'/') + ) + } + + /// Convert to a string representation + pub fn to_uri(&self) -> String { + if self.key.is_empty() { + format!("ipc://{}", self.bucket_address) + } else { + format!("ipc://{}/{}", self.bucket_address, self.key) + } + } +} + +/// Parse an address from a string (supports f/t addresses and 0x addresses) +pub fn parse_address(s: &str) -> Result

{ + // Try parsing as Filecoin address (f/t prefix) + let addr = Network::Mainnet + .parse_address(s) + .or_else(|e| match e { + NetworkError::UnknownNetwork => Network::Testnet.parse_address(s), + _ => Err(e), + }) + .or_else(|_| { + // Try parsing as Ethereum address (0x prefix) + let eth_addr = ethers::types::Address::from_str(s)?; + ipc_api::ethers_address_to_fil_address(ð_addr) + })?; + + Ok(addr) +} + +/// Check if a path string is a storage path (starts with ipc://) +pub fn is_storage_path(path: &str) -> bool { + path.starts_with("ipc://") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_storage_path() { + let path = StoragePath::parse("ipc://0x1234567890123456789012345678901234567890/documents/file.txt") + .expect("should parse"); + + assert_eq!(path.key, "documents/file.txt"); + } + + #[test] + fn test_parse_bucket_root() { + let path = StoragePath::parse("ipc://0x1234567890123456789012345678901234567890") + .expect("should parse"); + + assert!(path.is_bucket_root()); + assert_eq!(path.key, ""); + } + + #[test] + fn test_parse_bucket_with_trailing_slash() { + let path = StoragePath::parse("ipc://0x1234567890123456789012345678901234567890/") + .expect("should parse"); + + assert_eq!(path.key, ""); + } + + #[test] + fn test_invalid_uri() { + assert!(StoragePath::parse("http://bucket/file").is_err()); + assert!(StoragePath::parse("bucket/file").is_err()); + assert!(StoragePath::parse("ipc://").is_err()); + } + + #[test] + fn test_parent_path() { + let path = StoragePath::parse("ipc://0x1234567890123456789012345678901234567890/dir1/dir2/file.txt") + .expect("should parse"); + + assert_eq!(path.parent(), Some("dir1/dir2/".to_string())); + } + + #[test] + fn test_is_prefix_of() { + let path = StoragePath::parse("ipc://0x1234567890123456789012345678901234567890/documents/") + .expect("should parse"); + + assert!(path.is_prefix_of("documents/file.txt")); + assert!(path.is_prefix_of("documents/subfolder/file.txt")); + assert!(!path.is_prefix_of("other/file.txt")); + } + + #[test] + fn test_to_uri() { + let uri = "ipc://0x1234567890123456789012345678901234567890/path/to/file.txt"; + let path = StoragePath::parse(uri).expect("should parse"); + assert_eq!(path.to_uri(), uri); + } +} diff --git a/ipc/cli/src/commands/storage/rm.rs b/ipc/cli/src/commands/storage/rm.rs new file mode 100644 index 0000000000..bc5280a3c5 --- /dev/null +++ b/ipc/cli/src/commands/storage/rm.rs @@ -0,0 +1,248 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Remove command for deleting objects from storage + +use anyhow::{anyhow, Context, Result}; +use clap::Args; +use std::io::{self, Write}; +use std::path::PathBuf; + +use async_trait::async_trait; +use fendermint_rpc::client::FendermintClient; +use fendermint_rpc::message::SignedMessageFactory; +use fendermint_rpc::QueryClient; +use fendermint_vm_actor_interface::eam::EthAddress; +use fvm_shared::address::Address; +use fvm_shared::chainid::ChainID; + +use crate::commands::storage::{bucket, config::StorageConfig, path}; +use crate::{CommandLineHandler, GlobalArguments}; + +#[derive(Debug, Args)] +pub struct RemoveArgs { + /// Storage path (ipc://bucket_address/path/to/file) + #[arg(value_name = "PATH")] + pub path: String, + + /// Storage config file + #[arg(long)] + pub config: Option, + + /// Recursive delete (for prefix-based deletion) + #[arg(short, long)] + pub recursive: bool, + + /// Force deletion without confirmation + #[arg(short, long)] + pub force: bool, +} + +pub struct RemoveStorage; + +#[async_trait] +impl CommandLineHandler for RemoveStorage { + type Arguments = RemoveArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let storage_path = path::StoragePath::parse(&args.path)?; + + if storage_path.is_bucket_root() { + return Err(anyhow!("Cannot delete entire bucket. Specify a key or prefix.")); + } + + // Handle recursive deletion + if args.recursive { + return delete_recursive(&storage_path, args).await; + } + + // Single file deletion + delete_file(&storage_path, args).await + } +} + +async fn delete_file(storage_path: &path::StoragePath, args: &RemoveArgs) -> Result<()> { + // Confirm deletion unless --force + if !args.force { + print!("Delete {}? [y/N] ", storage_path.to_uri()); + io::stdout().flush()?; + + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + + if !input.trim().eq_ignore_ascii_case("y") { + println!("Aborted"); + return Ok(()); + } + } + + // Load config + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + // Create FendermintClient and bound client + let fm_client = FendermintClient::new_http( + config.tendermint_rpc_url.parse()?, + None, + )?; + + // Query chain ID from the network + let chain_id = bucket::query_chain_id(&fm_client) + .await + .context("Failed to query chain ID")?; + + let secret_key = SignedMessageFactory::read_secret_key( + &config.secret_key_file + )?; + + let pub_key = secret_key.public_key(); + let eth_addr = EthAddress::new_secp256k1(&pub_key.serialize()) + .context("failed to derive delegated address")?; + let addr = Address::new_delegated(10, ð_addr.0) + .context("failed to construct f410 address")?; + let state = fm_client + .actor_state(&addr, fendermint_vm_message::query::FvmQueryHeight::default()) + .await + .context("Failed to get actor state")?; + let sequence = state.value.map(|(_, s)| s.sequence).unwrap_or(0); + + let mf = SignedMessageFactory::new(secret_key, addr, sequence, ChainID::from(chain_id)); + let mut bound_client = fm_client.bind(mf); + + // Delete object + println!("Deleting {}...", storage_path.key); + + bucket::delete_object( + &mut bound_client, + storage_path.bucket_address, + storage_path.key.clone(), + ) + .await + .context("Failed to delete object")?; + + println!("✓ Deleted: {}", storage_path.key); + + Ok(()) +} + +async fn delete_recursive(storage_path: &path::StoragePath, args: &RemoveArgs) -> Result<()> { + // Load config + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + // List all objects with the prefix + let fm_client = FendermintClient::new_http( + config.tendermint_rpc_url.parse()?, + None, + )?; + + let prefix = storage_path.key.clone(); + let mut deleted_count = 0; + let mut start_key = None; + + loop { + let list_result = bucket::list_objects( + &fm_client, + storage_path.bucket_address, + Some(prefix.clone()), + None, // no delimiter for recursive + start_key, + 100, // batch size + ) + .await + .context("Failed to list objects")?; + + if list_result.objects.is_empty() { + break; + } + + // Confirm deletion unless --force + if !args.force && deleted_count == 0 { + println!("Found {} objects to delete with prefix: {}", list_result.objects.len(), prefix); + print!("Continue? [y/N] "); + io::stdout().flush()?; + + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + + if !input.trim().eq_ignore_ascii_case("y") { + println!("Aborted"); + return Ok(()); + } + } + + // Delete each object + let chain_id = bucket::query_chain_id(&fm_client) + .await + .context("Failed to query chain ID")?; + + let secret_key = SignedMessageFactory::read_secret_key( + &config.secret_key_file + )?; + + let pub_key = secret_key.public_key(); + let eth_addr = EthAddress::new_secp256k1(&pub_key.serialize()) + .context("failed to derive delegated address")?; + let addr = Address::new_delegated(10, ð_addr.0) + .context("failed to construct f410 address")?; + let state = fm_client + .actor_state(&addr, fendermint_vm_message::query::FvmQueryHeight::default()) + .await + .context("Failed to get actor state")?; + let sequence = state.value.map(|(_, s)| s.sequence).unwrap_or(0); + + let mf = SignedMessageFactory::new(secret_key, addr, sequence, ChainID::from(chain_id)); + let mut bound_client = fm_client.clone().bind(mf); + + for (key, _) in &list_result.objects { + let key_str = String::from_utf8_lossy(key).to_string(); + bucket::delete_object( + &mut bound_client, + storage_path.bucket_address, + key_str.clone(), + ) + .await + .with_context(|| format!("Failed to delete object: {}", key_str))?; + + println!("✓ Deleted: {}", key_str); + deleted_count += 1; + } + + // Check if there are more pages + if list_result.next_key.is_none() { + break; + } + + start_key = list_result.next_key.map(|k| String::from_utf8_lossy(&k).to_string()); + } + + println!("\nDeleted {} objects", deleted_count); + + Ok(()) +} diff --git a/ipc/cli/src/commands/storage/run.rs b/ipc/cli/src/commands/storage/run.rs new file mode 100644 index 0000000000..863776a7a0 --- /dev/null +++ b/ipc/cli/src/commands/storage/run.rs @@ -0,0 +1,319 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +use crate::commands::storage::config::{StorageConfig, StorageRunMode}; +use crate::CommandLineHandler; +use anyhow::{bail, Context, Result}; +use async_trait::async_trait; +use clap::Args; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::str::FromStr; + +pub(crate) struct RunStorage; + +#[async_trait] +impl CommandLineHandler for RunStorage { + type Arguments = RunStorageArgs; + + async fn handle(_global: &crate::GlobalArguments, args: &Self::Arguments) -> Result<()> { + let cfg = StorageConfig::load(&args.config).with_context(|| { + format!( + "failed to load storage config from {}", + args.config.display() + ) + })?; + + preflight(&cfg).await?; + + if args.register_operator { + run_register_operator(&cfg)?; + } + + let mode = args.mode.unwrap_or(cfg.run_mode); + match mode { + StorageRunMode::Node => run_node(&cfg), + StorageRunMode::Gateway => run_gateway(&cfg), + StorageRunMode::Both => run_both(&cfg), + } + } +} + +#[derive(Debug, Args)] +#[command(name = "run", about = "Run storage node and gateway")] +pub struct RunStorageArgs { + #[arg(long, help = "Path to storage YAML config")] + pub config: std::path::PathBuf, + #[arg(long, help = "Register node operator before launching services")] + pub register_operator: bool, + #[arg(long, help = "Override run mode (node|gateway|both)")] + pub mode: Option, +} + +fn run_register_operator(cfg: &StorageConfig) -> Result<()> { + log::info!("Registering storage node operator"); + let status = Command::new(&cfg.storage_node_bin) + .arg("register-operator") + .arg("--bls-key-file") + .arg(&cfg.bls_key_file) + .arg("--secret-key-file") + .arg(&cfg.secret_key_file) + .arg("--operator-rpc-url") + .arg(&cfg.operator_rpc_url) + .arg("--chain-rpc-url") + .arg(&cfg.tendermint_rpc_url) + .env("FM_NETWORK", &cfg.network) + .status() + .with_context(|| { + format!( + "failed to execute register-operator using {}", + cfg.storage_node_bin.display() + ) + })?; + + if !status.success() { + bail!("register-operator exited with status {}", status); + } + Ok(()) +} + +fn run_node(cfg: &StorageConfig) -> Result<()> { + log::info!("Starting storage node"); + let status = node_command(cfg)? + .status() + .context("failed to start storage node process")?; + if !status.success() { + bail!("storage node exited with status {}", status); + } + Ok(()) +} + +fn run_gateway(cfg: &StorageConfig) -> Result<()> { + log::info!("Starting storage gateway"); + let status = gateway_command(cfg)? + .status() + .context("failed to start storage gateway process")?; + if !status.success() { + bail!("storage gateway exited with status {}", status); + } + Ok(()) +} + +fn run_both(cfg: &StorageConfig) -> Result<()> { + log::info!("Starting storage gateway + storage node"); + let mut gateway = gateway_command(cfg)? + .spawn() + .context("failed to spawn storage gateway")?; + let mut node = node_command(cfg)? + .spawn() + .context("failed to spawn storage node")?; + + let node_status = node.wait().context("failed to wait for storage node")?; + if let Err(e) = terminate_child(&mut gateway) { + log::warn!("failed to stop gateway after node exit: {}", e); + } + + if !node_status.success() { + bail!("storage node exited with status {}", node_status); + } + Ok(()) +} + +fn terminate_child(child: &mut Child) -> Result<()> { + if child.try_wait()?.is_none() { + child.kill()?; + } + let _ = child.wait(); + Ok(()) +} + +fn node_command(cfg: &StorageConfig) -> Result { + let node_bin = resolve_bin_path(&cfg.storage_node_bin, "node")?; + let mut cmd = Command::new(&node_bin); + cmd.arg("run") + .arg("--secret-key-file") + .arg(&cfg.bls_key_file) + .arg("--iroh-path") + .arg(&cfg.iroh_node_path) + .arg("--rpc-url") + .arg(&cfg.tendermint_rpc_url) + .arg("--eth-rpc-url") + .arg(&cfg.eth_rpc_url) + .arg("--batch-size") + .arg(cfg.node_batch_size.to_string()) + .arg("--poll-interval-secs") + .arg(cfg.node_poll_interval_secs.to_string()) + .arg("--max-concurrent-downloads") + .arg(cfg.node_max_concurrent_downloads.to_string()) + .arg("--rpc-bind-addr") + .arg(&cfg.node_rpc_bind_addr) + .env("FM_NETWORK", &cfg.network) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + if let Some(v4) = &cfg.iroh_node_v4_addr { + cmd.arg("--iroh-v4-addr").arg(v4); + } + Ok(cmd) +} + +fn gateway_command(cfg: &StorageConfig) -> Result { + let gateway_bin = resolve_bin_path(&cfg.storage_gateway_bin, "gateway")?; + let mut cmd = Command::new(&gateway_bin); + cmd.arg("--secret-key-file") + .arg(&cfg.secret_key_file) + .arg("--bls-key-file") + .arg(&cfg.bls_key_file) + .arg("--rpc-url") + .arg(&cfg.tendermint_rpc_url) + .arg("--objects-listen-addr") + .arg(&cfg.objects_listen_addr) + .arg("--iroh-path") + .arg(&cfg.iroh_gateway_path) + .env("FM_NETWORK", &cfg.network) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + if let Some(v4) = &cfg.iroh_gateway_v4_addr { + cmd.arg("--iroh-v4-addr").arg(v4); + } + Ok(cmd) +} + +async fn preflight(cfg: &StorageConfig) -> Result<()> { + if resolve_bin_path(&cfg.storage_node_bin, "node").is_err() { + bail!( + "storage node binary not found at {} (build: cargo build --release -p ipc-decentralized-storage --bin node --bin gateway)", + cfg.storage_node_bin.display() + ); + } + if resolve_bin_path(&cfg.storage_gateway_bin, "gateway").is_err() { + bail!( + "storage gateway binary not found at {}", + cfg.storage_gateway_bin.display() + ); + } + if !cfg.secret_key_file.exists() { + bail!( + "operator secret key file not found: {}", + cfg.secret_key_file.display() + ); + } + if !cfg.bls_key_file.exists() { + if let Some(parent) = cfg.bls_key_file.parent() { + tokio::fs::create_dir_all(parent).await.with_context(|| { + format!( + "failed to create BLS key directory at {}", + parent.display() + ) + })?; + } + let node_bin = resolve_bin_path(&cfg.storage_node_bin, "node")?; + log::info!( + "BLS key file not found at {}; generating one now", + cfg.bls_key_file.display() + ); + let status = Command::new(&node_bin) + .arg("generate-bls-key") + .arg("--output") + .arg(&cfg.bls_key_file) + .status() + .with_context(|| { + format!( + "failed to generate BLS key via {}", + node_bin.display() + ) + })?; + if !status.success() { + bail!( + "failed to generate BLS key at {} (exit status {})", + cfg.bls_key_file.display(), + status + ); + } + log::info!("Generated BLS key at {}", cfg.bls_key_file.display()); + } + if cfg + .secret_key_file + .ends_with(std::path::Path::new("fendermint/validator.sk")) + { + log::warn!( + "Storage config uses validator key ({}). If register-operator fails with sender/account errors, regenerate config with --secret-key-file pointing to a dedicated funded operator key.", + cfg.secret_key_file.display() + ); + } + + if !cfg.node_home.exists() { + bail!( + "node home directory does not exist: {}", + cfg.node_home.display() + ); + } + + if !cfg.iroh_node_path.exists() { + tokio::fs::create_dir_all(&cfg.iroh_node_path) + .await + .with_context(|| { + format!( + "failed to create node iroh directory at {}", + cfg.iroh_node_path.display() + ) + })?; + } + if !cfg.iroh_gateway_path.exists() { + tokio::fs::create_dir_all(&cfg.iroh_gateway_path) + .await + .with_context(|| { + format!( + "failed to create gateway iroh directory at {}", + cfg.iroh_gateway_path.display() + ) + })?; + } + Ok(()) +} + +fn resolve_bin_path(configured: &Path, bin_name: &str) -> Result { + if configured.exists() { + return Ok(configured.to_path_buf()); + } + + let fallback = workspace_root().join("target/release").join(bin_name); + if fallback.exists() { + log::warn!( + "Configured {} binary not found at {}; using fallback {}", + bin_name, + configured.display(), + fallback.display() + ); + return Ok(fallback); + } + + bail!( + "{} binary not found at {} or fallback {}", + bin_name, + configured.display(), + fallback.display() + ) +} + +fn workspace_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|p| p.parent()) + .unwrap_or_else(|| Path::new(".")) + .to_path_buf() +} + +impl FromStr for StorageRunMode { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "node" => Ok(Self::Node), + "gateway" => Ok(Self::Gateway), + "both" => Ok(Self::Both), + _ => bail!("invalid run mode '{}', expected node|gateway|both", s), + } + } +} diff --git a/ipc/cli/src/commands/storage/stat.rs b/ipc/cli/src/commands/storage/stat.rs new file mode 100644 index 0000000000..610f02f029 --- /dev/null +++ b/ipc/cli/src/commands/storage/stat.rs @@ -0,0 +1,143 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Stat command for displaying object metadata from storage + +use anyhow::{anyhow, Context, Result}; +use clap::Args; +use std::path::PathBuf; + +use fendermint_rpc::client::FendermintClient; +use serde_json::json; + +use async_trait::async_trait; + +use crate::commands::storage::{bucket, config::StorageConfig, path}; +use crate::{CommandLineHandler, GlobalArguments}; + +#[derive(Debug, Args)] +pub struct StatArgs { + /// Storage path (ipc://bucket_address/path/to/file) + #[arg(value_name = "PATH")] + pub path: String, + + /// Storage config file + #[arg(long)] + pub config: Option, + + /// Output in JSON format + #[arg(long)] + pub json: bool, +} + +pub struct StatStorage; + +#[async_trait] +impl CommandLineHandler for StatStorage { + type Arguments = StatArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let storage_path = path::StoragePath::parse(&args.path)?; + + if storage_path.is_bucket_root() { + return Err(anyhow!("Path must include a file key, not just bucket address")); + } + + // Load config + let config_path = args.config.clone().unwrap_or_else(|| { + dirs::home_dir() + .unwrap() + .join(".ipc") + .join("storage.yaml") + }); + + let config = if config_path.exists() { + StorageConfig::load(&config_path)? + } else { + return Err(anyhow!( + "Storage config not found at {}. Run 'ipc-cli storage init' first.", + config_path.display() + )); + }; + + // Create FendermintClient + let mut fm_client = FendermintClient::new_http( + config.tendermint_rpc_url.parse()?, + None, + )?; + + // Get object metadata + let object = bucket::get_object( + &mut fm_client, + storage_path.bucket_address, + storage_path.key.clone(), + ) + .await + .context("Failed to get object")?; + + match object { + Some(obj) => { + if args.json { + print_json(&storage_path, &obj)?; + } else { + print_table(&storage_path, &obj)?; + } + } + None => { + return Err(anyhow!("Object not found: {}", storage_path.key)); + } + } + + Ok(()) + } +} + +fn print_json(storage_path: &path::StoragePath, obj: &fendermint_actor_bucket::Object) -> Result<()> { + let output = json!({ + "bucket": storage_path.bucket_address.to_string(), + "key": storage_path.key, + "hash": format!("0x{}", hex::encode(obj.hash.0)), + "recovery_hash": format!("0x{}", hex::encode(obj.recovery_hash.0)), + "size": obj.size, + "expiry": obj.expiry, + "metadata": obj.metadata, + }); + + println!("{}", serde_json::to_string_pretty(&output)?); + Ok(()) +} + +fn print_table(storage_path: &path::StoragePath, obj: &fendermint_actor_bucket::Object) -> Result<()> { + println!("Object: {}", storage_path.to_uri()); + println!(" Bucket: {}", storage_path.bucket_address); + println!(" Key: {}", storage_path.key); + println!(" Hash: 0x{}", hex::encode(obj.hash.0)); + println!(" Recovery Hash: 0x{}", hex::encode(obj.recovery_hash.0)); + println!(" Size: {} bytes ({})", obj.size, format_size(obj.size)); + println!(" Expiry: block {}", obj.expiry); + + if !obj.metadata.is_empty() { + println!(" Metadata:"); + for (key, value) in &obj.metadata { + println!(" {}: {}", key, value); + } + } + + Ok(()) +} + +fn format_size(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + + if bytes >= GB { + format!("{:.2} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.2} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.2} KB", bytes as f64 / KB as f64) + } else { + format!("{} B", bytes) + } +} diff --git a/ipc/cli/src/commands/storage/sync.rs b/ipc/cli/src/commands/storage/sync.rs new file mode 100644 index 0000000000..343c2e4240 --- /dev/null +++ b/ipc/cli/src/commands/storage/sync.rs @@ -0,0 +1,97 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: MIT + +//! Sync command for synchronizing directories with storage + +use anyhow::{anyhow, Result}; +use clap::Args; +use std::path::PathBuf; + +use async_trait::async_trait; + +use crate::commands::storage::path; +use crate::{CommandLineHandler, GlobalArguments}; + +#[derive(Debug, Args)] +pub struct SyncArgs { + /// Source path (local directory or ipc://bucket/prefix) + #[arg(value_name = "SOURCE")] + pub source: String, + + /// Destination path (ipc://bucket/prefix or local directory) + #[arg(value_name = "DEST")] + pub dest: String, + + /// Gateway URL (overrides config and env var) + #[arg(long)] + pub gateway: Option, + + /// Storage config file + #[arg(long)] + pub config: Option, + + /// Dry run (show what would be synced) + #[arg(long)] + pub dry_run: bool, + + /// Delete files in destination that don't exist in source + #[arg(long)] + pub delete: bool, +} + +pub struct SyncStorage; + +#[async_trait] +impl CommandLineHandler for SyncStorage { + type Arguments = SyncArgs; + + async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> { + let source_is_storage = path::is_storage_path(&args.source); + let dest_is_storage = path::is_storage_path(&args.dest); + + match (source_is_storage, dest_is_storage) { + (false, true) => { + // Local -> Storage sync + sync_local_to_storage(args).await + } + (true, false) => { + // Storage -> Local sync + sync_storage_to_local(args).await + } + (true, true) => { + // Storage -> Storage sync + Err(anyhow!("Syncing between storage locations not yet implemented")) + } + (false, false) => { + Err(anyhow!( + "At least one path must be a storage path (ipc://...)" + )) + } + } + } +} + +async fn sync_local_to_storage(_args: &SyncArgs) -> Result<()> { + // TODO: Implement by: + // 1. List all local files in source directory + // 2. List all storage objects with destination prefix + // 3. Compare and determine: + // - New files to upload + // - Modified files to re-upload (compare size/hash) + // - Files to delete (if --delete flag) + // 4. Perform operations (unless --dry-run) + + Err(anyhow!( + "Sync functionality not yet implemented.\n\ + Use 'ipc-cli storage cp -r' for recursive upload/download." + )) +} + +async fn sync_storage_to_local(_args: &SyncArgs) -> Result<()> { + // TODO: Similar to sync_local_to_storage but in reverse + + Err(anyhow!( + "Sync functionality not yet implemented.\n\ + Use 'ipc-cli storage cp -r' for recursive upload/download." + )) +} diff --git a/ipc/cli/src/main.rs b/ipc/cli/src/main.rs index 6fda1b955b..61c1c922b9 100644 --- a/ipc/cli/src/main.rs +++ b/ipc/cli/src/main.rs @@ -42,34 +42,15 @@ fn print_user_friendly_error(error: &anyhow::Error) { } fn extract_meaningful_error(error: &anyhow::Error) -> String { - // Get the root cause of the error chain - let mut root_cause = error.to_string(); - - // Get the first source error if available - if let Some(source) = error.source() { - root_cause = source.to_string(); - } - - // Clean up common error patterns - let cleaned = root_cause - .replace("error processing command Some(", "") - .replace("main process failed: ", "") - .trim() - .to_string(); - - // Special handling for contract revert errors - if cleaned.contains("Contract call reverted with data:") { - // Provide a generic but helpful message - return "Contract operation failed. The transaction was reverted by the smart contract." - .to_string(); - } - - // If the cleaned message is significantly shorter, use it - if cleaned.len() < root_cause.len() * 2 / 3 { - cleaned - } else { - root_cause + // Show the full error chain so no details are lost + let mut parts = Vec::new(); + parts.push(error.to_string()); + let mut source = error.source(); + while let Some(s) = source { + parts.push(s.to_string()); + source = s.source(); } + parts.join(": ") } fn is_contract_related_error(error_msg: &str) -> bool {