From 89306f6e923fd6f7931142fde8b4761926f871a4 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Wed, 11 Feb 2026 16:12:08 -0800 Subject: [PATCH 1/3] Add support for anonymous Azure blob access The current support for Azure blob storage can't deal with the situation where the blob store needs to be accessed anonymously. This PR adds the opendal http backend and an environment variable that can be used to enable Azure anonymous access. If the environment variable is set, sccache falls back to direct http access instead of the opendal authentication flow. --- Cargo.toml | 2 +- docs/Azure.md | 4 ++ docs/Configuration.md | 3 ++ src/cache/azure.rs | 87 ++++++++++++++++++++++++++++++++++++++++++- src/cache/cache.rs | 3 +- src/config.rs | 32 ++++++++++++++++ 6 files changed, 128 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cc3f13d85..f83b9af9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -167,7 +167,7 @@ all = [ "oss", "cos", ] -azure = ["opendal/services-azblob", "reqsign", "reqwest"] +azure = ["opendal/services-azblob", "opendal/services-http", "reqsign", "reqwest"] cos = ["opendal/services-cos", "reqsign", "reqwest"] default = ["all"] gcs = ["opendal/services-gcs", "reqsign", "url", "reqwest"] diff --git a/docs/Azure.md b/docs/Azure.md index c218defd2..016e47616 100644 --- a/docs/Azure.md +++ b/docs/Azure.md @@ -6,4 +6,8 @@ the container for you - you'll need to do that yourself. You can also define a prefix that will be prepended to the keys of all cache objects created and read within the container, effectively creating a scope. To do that use the `SCCACHE_AZURE_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application. +Alternatively, the `SCCACHE_AZURE_NO_CREDENTIALS` environment variable can be set to use public readonly access to the Azure Blob Storage container, without the need for credentials. Valid values for this environment variable are `true`, `1`, `false`, and `0`. + +When using anonymous access, the connection string only needs to contain the endpoint, e.g. `BlobEndpoint=https://accountname.blob.core.windows.net`. + **Important:** The environment variables are only taken into account when the server starts, i.e. only on the first run. diff --git a/docs/Configuration.md b/docs/Configuration.md index 57f3682e6..b651399c8 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -47,6 +47,8 @@ connection_string = "BlobEndpoint=https://example.blob.core.windows.net/;SharedA container = "my_container_name" # Optional string to prepend to each blob storage key key_prefix = "" +# Set to true to use anonymous access (no credentials required) +no_credentials = false [cache.disk] dir = "/tmp/.cache/sccache" @@ -236,6 +238,7 @@ The full url appears then as `redis://user:passwd@1.2.3.4:6379/?db=1`. * `SCCACHE_AZURE_CONNECTION_STRING` * `SCCACHE_AZURE_BLOB_CONTAINER` * `SCCACHE_AZURE_KEY_PREFIX` +* `SCCACHE_AZURE_NO_CREDENTIALS` #### gha diff --git a/src/cache/azure.rs b/src/cache/azure.rs index 7b5a83b47..651c41e91 100644 --- a/src/cache/azure.rs +++ b/src/cache/azure.rs @@ -17,6 +17,7 @@ use opendal::Operator; use opendal::layers::{HttpClientLayer, LoggingLayer}; use opendal::services::Azblob; +use opendal::services::Http; use crate::errors::*; @@ -24,8 +25,37 @@ use super::http_client::set_user_agent; pub struct AzureBlobCache; +/// Parse the `BlobEndpoint` value from an Azure Storage connection string. +fn blob_endpoint_from_connection_string(connection_string: &str) -> Result { + for part in connection_string.split(';') { + let part = part.trim(); + if let Some(value) = part.strip_prefix("BlobEndpoint=") { + return Ok(value.to_string()); + } + } + bail!("connection string does not contain a BlobEndpoint") +} + impl AzureBlobCache { - pub fn build(connection_string: &str, container: &str, key_prefix: &str) -> Result { + pub fn build( + connection_string: &str, + container: &str, + key_prefix: &str, + no_credentials: bool, + ) -> Result { + if no_credentials { + Self::build_http_readonly(connection_string, container, key_prefix) + } else { + Self::build_azblob(connection_string, container, key_prefix) + } + } + + /// Build an operator using the OpenDAL Azblob service (authenticated). + fn build_azblob( + connection_string: &str, + container: &str, + key_prefix: &str, + ) -> Result { let builder = Azblob::from_connection_string(connection_string)? .container(container) .root(key_prefix); @@ -36,4 +66,59 @@ impl AzureBlobCache { .finish(); Ok(op) } + + /// Build an operator using the OpenDAL HTTP service for anonymous + /// read-only access. The endpoint is constructed from the connection + /// string's `BlobEndpoint` value plus the container name, so that + /// reads go directly to + /// `https://.blob.core.windows.net//`. + fn build_http_readonly( + connection_string: &str, + container: &str, + key_prefix: &str, + ) -> Result { + let blob_endpoint = blob_endpoint_from_connection_string(connection_string)?; + let endpoint = format!( + "{}/{}", + blob_endpoint.trim_end_matches('/'), + container + ); + + let builder = Http::default().endpoint(&endpoint).root(key_prefix); + + let op = Operator::new(builder)? + .layer(HttpClientLayer::new(set_user_agent())) + .layer(LoggingLayer::default()) + .finish(); + Ok(op) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_blob_endpoint() { + let cs = "BlobEndpoint=https://myaccount.blob.core.windows.net"; + assert_eq!( + blob_endpoint_from_connection_string(cs).unwrap(), + "https://myaccount.blob.core.windows.net" + ); + } + + #[test] + fn test_parse_blob_endpoint_from_full_connection_string() { + let cs = "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=abc123;BlobEndpoint=https://myaccount.blob.core.windows.net"; + assert_eq!( + blob_endpoint_from_connection_string(cs).unwrap(), + "https://myaccount.blob.core.windows.net" + ); + } + + #[test] + fn test_parse_blob_endpoint_missing() { + let cs = "DefaultEndpointsProtocol=https;AccountName=myaccount"; + assert!(blob_endpoint_from_connection_string(cs).is_err()); + } } diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 906c69f5f..c4e2371aa 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -305,9 +305,10 @@ pub fn build_single_cache( connection_string, container, key_prefix, + no_credentials, }) => { debug!("Init azure cache with container {container}, key_prefix {key_prefix}"); - let operator = AzureBlobCache::build(connection_string, container, key_prefix) + let operator = AzureBlobCache::build(connection_string, container, key_prefix, *no_credentials) .map_err(|err| anyhow!("create azure cache failed: {err:?}"))?; let storage = RemoteStorage::new(operator, basedirs.to_vec()); Ok(Arc::new(storage)) diff --git a/src/config.rs b/src/config.rs index aeeca7f2f..4d94def1a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -185,7 +185,10 @@ impl HTTPUrl { pub struct AzureCacheConfig { pub connection_string: String, pub container: String, + #[serde(default)] pub key_prefix: String, + #[serde(default)] + pub no_credentials: bool, } /// Configuration switches for preprocessor cache mode. @@ -910,10 +913,12 @@ fn config_from_env() -> Result { env::var("SCCACHE_AZURE_BLOB_CONTAINER"), ) { let key_prefix = key_prefix_from_env_var("SCCACHE_AZURE_KEY_PREFIX"); + let no_credentials = bool_from_env_var("SCCACHE_AZURE_NO_CREDENTIALS")?.unwrap_or(false); Some(AzureCacheConfig { connection_string, container, key_prefix, + no_credentials, }) } else { None @@ -1429,6 +1434,7 @@ fn config_overrides() { connection_string: String::new(), container: String::new(), key_prefix: String::new(), + no_credentials: false, }), disk: Some(DiskCacheConfig { dir: "/env-cache".into(), @@ -2597,3 +2603,29 @@ fn test_integration_env_variable_to_strip() { let output2 = strip_basedirs(input2, &config.basedirs); assert_eq!(&*output2, b"# 1 \"obj/file.o\""); } + +#[test] +fn test_azure_config_deserializes_without_optional_fields() { + let toml_str = r#" +connection_string = "DefaultEndpointsProtocol=https;AccountName=test" +container = "my-container" +"#; + let config: AzureCacheConfig = toml::from_str(toml_str).expect("should deserialize"); + assert_eq!(config.connection_string, "DefaultEndpointsProtocol=https;AccountName=test"); + assert_eq!(config.container, "my-container"); + assert_eq!(config.key_prefix, ""); + assert!(!config.no_credentials); +} + +#[test] +fn test_azure_config_deserializes_with_all_fields() { + let toml_str = r#" +connection_string = "DefaultEndpointsProtocol=https;AccountName=test" +container = "my-container" +key_prefix = "prefix/" +no_credentials = true +"#; + let config: AzureCacheConfig = toml::from_str(toml_str).expect("should deserialize"); + assert_eq!(config.key_prefix, "prefix/"); + assert!(config.no_credentials); +} From 53b3c7eec6d350e1fd15054cb4ccee2bcc82f91e Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Thu, 12 Feb 2026 10:43:27 -0800 Subject: [PATCH 2/3] Fix formatting --- src/cache/azure.rs | 6 +----- src/cache/cache.rs | 5 +++-- src/config.rs | 5 ++++- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cache/azure.rs b/src/cache/azure.rs index 651c41e91..1fa357eea 100644 --- a/src/cache/azure.rs +++ b/src/cache/azure.rs @@ -78,11 +78,7 @@ impl AzureBlobCache { key_prefix: &str, ) -> Result { let blob_endpoint = blob_endpoint_from_connection_string(connection_string)?; - let endpoint = format!( - "{}/{}", - blob_endpoint.trim_end_matches('/'), - container - ); + let endpoint = format!("{}/{}", blob_endpoint.trim_end_matches('/'), container); let builder = Http::default().endpoint(&endpoint).root(key_prefix); diff --git a/src/cache/cache.rs b/src/cache/cache.rs index c4e2371aa..51c5fa35f 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -308,8 +308,9 @@ pub fn build_single_cache( no_credentials, }) => { debug!("Init azure cache with container {container}, key_prefix {key_prefix}"); - let operator = AzureBlobCache::build(connection_string, container, key_prefix, *no_credentials) - .map_err(|err| anyhow!("create azure cache failed: {err:?}"))?; + let operator = + AzureBlobCache::build(connection_string, container, key_prefix, *no_credentials) + .map_err(|err| anyhow!("create azure cache failed: {err:?}"))?; let storage = RemoteStorage::new(operator, basedirs.to_vec()); Ok(Arc::new(storage)) } diff --git a/src/config.rs b/src/config.rs index 4d94def1a..43a21da16 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2611,7 +2611,10 @@ connection_string = "DefaultEndpointsProtocol=https;AccountName=test" container = "my-container" "#; let config: AzureCacheConfig = toml::from_str(toml_str).expect("should deserialize"); - assert_eq!(config.connection_string, "DefaultEndpointsProtocol=https;AccountName=test"); + assert_eq!( + config.connection_string, + "DefaultEndpointsProtocol=https;AccountName=test" + ); assert_eq!(config.container, "my-container"); assert_eq!(config.key_prefix, ""); assert!(!config.no_credentials); From 4914aeaca807c9c3b0f624e9483a35463d256189 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Thu, 12 Feb 2026 10:55:45 -0800 Subject: [PATCH 3/3] Add some tests for building with anonymous access --- src/cache/azure.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/cache/azure.rs b/src/cache/azure.rs index 1fa357eea..6dc5328ee 100644 --- a/src/cache/azure.rs +++ b/src/cache/azure.rs @@ -117,4 +117,20 @@ mod tests { let cs = "DefaultEndpointsProtocol=https;AccountName=myaccount"; assert!(blob_endpoint_from_connection_string(cs).is_err()); } + + #[test] + fn test_build_no_credentials() { + let cs = "BlobEndpoint=https://myaccount.blob.core.windows.net"; + let op = AzureBlobCache::build(cs, "mycontainer", "/prefix", true).unwrap(); + let info = op.info(); + assert_eq!(info.scheme(), "http"); + assert_eq!(info.root(), "/prefix/"); + } + + #[test] + fn test_build_no_credentials_missing_endpoint() { + let cs = "AccountName=myaccount"; + let op = AzureBlobCache::build(cs, "mycontainer", "/prefix", true); + assert!(op.is_err()); + } }