Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 6 additions & 70 deletions src/cache/cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,6 @@ use crate::cache::oss::OSSCache;
use crate::cache::redis::RedisCache;
#[cfg(feature = "s3")]
use crate::cache::s3::S3Cache;
#[cfg(feature = "webdav")]
use crate::cache::webdav::WebdavCache;
use crate::compiler::PreprocessorCacheEntry;
use crate::config::Config;
#[cfg(any(
feature = "azure",
feature = "gcs",
Expand All @@ -45,10 +41,14 @@ use crate::config::Config;
feature = "oss",
feature = "cos"
))]
use crate::config::{self, CacheType};
use crate::cache::utils::normalize_key;
#[cfg(feature = "webdav")]
use crate::cache::webdav::WebdavCache;
use crate::compiler::PreprocessorCacheEntry;
use crate::config::Config;
use crate::config::{self, CacheType, PreprocessorCacheModeConfig};
use async_trait::async_trait;

use serde::{Deserialize, Serialize};
use std::io;
use std::sync::Arc;
use std::time::Duration;
Expand Down Expand Up @@ -134,57 +134,6 @@ pub trait Storage: Send + Sync {
}
}

/// Configuration switches for preprocessor cache mode.
#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
#[serde(default)]
pub struct PreprocessorCacheModeConfig {
/// Whether to use preprocessor cache mode entirely
pub use_preprocessor_cache_mode: bool,
/// If false (default), only compare header files by hashing their contents.
/// If true, will use size + ctime + mtime to check whether a file has changed.
/// See other flags below for more control over this behavior.
pub file_stat_matches: bool,
/// If true (default), uses the ctime (file status change on UNIX,
/// creation time on Windows) to check that a file has/hasn't changed.
/// Can be useful to disable when backdating modification times
/// in a controlled manner.
pub use_ctime_for_stat: bool,
/// If true, ignore `__DATE__`, `__TIME__` and `__TIMESTAMP__` being present
/// in the source code. Will speed up preprocessor cache mode,
/// but can result in false positives.
pub ignore_time_macros: bool,
/// If true, preprocessor cache mode will not cache system headers, only
/// add them to the hash.
pub skip_system_headers: bool,
/// If true (default), will add the current working directory in the hash to
/// distinguish two compilations from different directories.
pub hash_working_directory: bool,
}

impl Default for PreprocessorCacheModeConfig {
fn default() -> Self {
Self {
use_preprocessor_cache_mode: false,
file_stat_matches: false,
use_ctime_for_stat: true,
ignore_time_macros: false,
skip_system_headers: false,
hash_working_directory: true,
}
}
}

impl PreprocessorCacheModeConfig {
/// Return a default [`Self`], but with the cache active.
pub fn activated() -> Self {
Self {
use_preprocessor_cache_mode: true,
..Default::default()
}
}
}

/// Wrapper for opendal::Operator that adds basedirs support
#[cfg(any(
feature = "azure",
Expand Down Expand Up @@ -332,11 +281,6 @@ impl Storage for RemoteStorage {
}
}

/// Normalize key `abcdef` into `a/b/c/abcdef`
pub(in crate::cache) fn normalize_key(key: &str) -> String {
format!("{}/{}/{}/{}", &key[0..1], &key[1..2], &key[2..3], &key)
}

/// Build a single cache storage from CacheType
/// Helper function used by storage_from_config for both single and multi-level caches
#[cfg(any(
Expand Down Expand Up @@ -587,14 +531,6 @@ mod test {
use crate::config::CacheModeConfig;
use fs_err as fs;

#[test]
fn test_normalize_key() {
assert_eq!(
normalize_key("0123456789abcdef0123456789abcdef"),
"0/1/2/0123456789abcdef0123456789abcdef"
);
}

#[test]
fn test_read_write_mode_local() {
let runtime = tokio::runtime::Builder::new_current_thread()
Expand Down
3 changes: 2 additions & 1 deletion src/cache/disk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ use std::time::{Duration, Instant};
use crate::errors::*;

use super::lazy_disk_cache::LazyDiskCache;
use super::{PreprocessorCacheModeConfig, normalize_key};
use super::utils::normalize_key;
use crate::config::PreprocessorCacheModeConfig;

/// A cache that stores entries at local disk paths.
pub struct DiskCache {
Expand Down
3 changes: 1 addition & 2 deletions src/cache/readonly.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@ use async_trait::async_trait;

use crate::cache::{Cache, CacheMode, CacheWrite, Storage};
use crate::compiler::PreprocessorCacheEntry;
use crate::config::PreprocessorCacheModeConfig;
use crate::errors::*;

use super::PreprocessorCacheModeConfig;

pub struct ReadOnlyStorage(pub Arc<dyn Storage>);

#[async_trait]
Expand Down
18 changes: 18 additions & 0 deletions src/cache/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ use std::path::Path;

use crate::errors::*;

/// Normalize key `abcdef` into `a/b/c/abcdef`
pub(in crate::cache) fn normalize_key(key: &str) -> String {
format!("{}/{}/{}/{}", &key[0..1], &key[1..2], &key[2..3], &key)
}

#[cfg(unix)]
pub(in crate::cache) fn get_file_mode(file: &fs::File) -> Result<Option<u32>> {
use std::os::unix::fs::MetadataExt;
Expand All @@ -42,3 +47,16 @@ pub(in crate::cache) fn set_file_mode(path: &Path, mode: u32) -> Result<()> {
pub(in crate::cache) fn set_file_mode(_path: &Path, _mode: u32) -> Result<()> {
Ok(())
}

#[cfg(test)]
mod test {
use super::*;

#[test]
fn test_normalize_key() {
assert_eq!(
normalize_key("0123456789abcdef0123456789abcdef"),
"0/1/2/0123456789abcdef0123456789abcdef"
);
}
}
3 changes: 2 additions & 1 deletion src/compiler/c.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::cache::{FileObjectSource, PreprocessorCacheModeConfig, Storage};
use crate::cache::{FileObjectSource, Storage};
use crate::compiler::preprocessor_cache::preprocessor_cache_entry_hash_key;
use crate::compiler::{
Cacheable, ColorMode, Compilation, CompileCommand, Compiler, CompilerArguments, CompilerHasher,
CompilerKind, HashResult, Language,
};
#[cfg(feature = "dist-client")]
use crate::compiler::{DistPackagers, NoopOutputsRewriter};
use crate::config::PreprocessorCacheModeConfig;
use crate::dist;
#[cfg(feature = "dist-client")]
use crate::dist::pkg;
Expand Down
3 changes: 2 additions & 1 deletion src/compiler/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1882,7 +1882,8 @@ where
mod test {
use super::*;
use crate::cache::disk::DiskCache;
use crate::cache::{CacheMode, CacheRead, PreprocessorCacheModeConfig};
use crate::cache::{CacheMode, CacheRead};
use crate::config::PreprocessorCacheModeConfig;
use crate::mock_command::*;
use crate::test::mock_storage::MockStorage;
use crate::test::utils::*;
Expand Down
2 changes: 1 addition & 1 deletion src/compiler/preprocessor_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ use chrono::Datelike;
use serde::{Deserialize, Serialize};

use crate::{
cache::PreprocessorCacheModeConfig,
config::PreprocessorCacheModeConfig,
util::{Digest, HashToDigest, MetadataCtimeExt, Timestamp, encode_path, strip_basedirs},
};

Expand Down
52 changes: 51 additions & 1 deletion src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ use std::sync::{LazyLock, Mutex};
use std::{collections::HashMap, fmt};
use typed_path::Utf8TypedPathBuf;

pub use crate::cache::PreprocessorCacheModeConfig;
use crate::errors::*;

static CACHED_CONFIG_PATH: LazyLock<PathBuf> = LazyLock::new(CachedConfig::file_config_path);
Expand Down Expand Up @@ -189,6 +188,57 @@ pub struct AzureCacheConfig {
pub key_prefix: String,
}

/// Configuration switches for preprocessor cache mode.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
#[serde(default)]
pub struct PreprocessorCacheModeConfig {
/// Whether to use preprocessor cache mode entirely
pub use_preprocessor_cache_mode: bool,
/// If false (default), only compare header files by hashing their contents.
/// If true, will use size + ctime + mtime to check whether a file has changed.
/// See other flags below for more control over this behavior.
pub file_stat_matches: bool,
/// If true (default), uses the ctime (file status change on UNIX,
/// creation time on Windows) to check that a file has/hasn't changed.
/// Can be useful to disable when backdating modification times
/// in a controlled manner.
pub use_ctime_for_stat: bool,
/// If true, ignore `__DATE__`, `__TIME__` and `__TIMESTAMP__` being present
/// in the source code. Will speed up preprocessor cache mode,
/// but can result in false positives.
pub ignore_time_macros: bool,
/// If true, preprocessor cache mode will not cache system headers, only
/// add them to the hash.
pub skip_system_headers: bool,
/// If true (default), will add the current working directory in the hash to
/// distinguish two compilations from different directories.
pub hash_working_directory: bool,
}

impl Default for PreprocessorCacheModeConfig {
fn default() -> Self {
Self {
use_preprocessor_cache_mode: false,
file_stat_matches: false,
use_ctime_for_stat: true,
ignore_time_macros: false,
skip_system_headers: false,
hash_working_directory: true,
}
}
}

impl PreprocessorCacheModeConfig {
/// Return a default [`Self`], but with the cache active.
pub fn activated() -> Self {
Self {
use_preprocessor_cache_mode: true,
..Default::default()
}
}
}

#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
#[serde(default)]
Expand Down
3 changes: 2 additions & 1 deletion src/test/mock_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::cache::{Cache, CacheWrite, PreprocessorCacheModeConfig, Storage};
use crate::cache::{Cache, CacheWrite, Storage};
use crate::config::PreprocessorCacheModeConfig;
use crate::errors::*;
use async_trait::async_trait;
use futures::channel::mpsc;
Expand Down
3 changes: 2 additions & 1 deletion src/test/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::cache::CacheMode;
use crate::cache::disk::DiskCache;
use crate::cache::{CacheMode, PreprocessorCacheModeConfig};
use crate::client::connect_to_server;
use crate::commands::{do_compile, request_shutdown, request_stats};
use crate::config::PreprocessorCacheModeConfig;
use crate::jobserver::Client;
use crate::mock_command::*;
use crate::server::{DistClientContainer, SccacheServer, ServerMessage};
Expand Down
Loading