Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/performance-regression.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
check:
name: Check expected performance
runs-on: ubuntu-latest
timeout-minutes: 60
timeout-minutes: 90
env:
BASE: ${{ inputs.base || 'anchor' }}
DRIFT_ANCHOR_FILE: src/performance-harness/drift-anchor.txt
Expand Down
36 changes: 18 additions & 18 deletions src/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 11 additions & 11 deletions src/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ resolver = "2"
name = "cloudformation-validate"

[workspace.package]
version = "1.14.0"
version = "1.15.0"
edition = "2024"
rust-version = "1.96"
license = "Apache-2.0"
Expand All @@ -46,16 +46,16 @@ categories = [
]

[workspace.dependencies]
cel-engine = { package = "cloudformation-validate-cel-engine", path = "cel-engine", version = "=1.14.0" }
composite-engine = { package = "cloudformation-validate-composite-engine", path = "composite-engine", version = "=1.14.0" }
data-source = { package = "cloudformation-validate-data-source", path = "data-source", version = "=1.14.0", default-features = false }
diagnostics = { package = "cloudformation-validate-diagnostics", path = "diagnostics", version = "=1.14.0" }
guard-translator = { package = "cloudformation-validate-guard-translator", path = "guard-translator", version = "=1.14.0" }
rego-engine = { package = "cloudformation-validate-rego-engine", path = "rego-engine", version = "=1.14.0" }
rules = { package = "cloudformation-validate-rules", path = "rules", version = "=1.14.0" }
schema-validator = { package = "cloudformation-validate-schema-validator", path = "schema-validator", version = "=1.14.0" }
template-model = { package = "cloudformation-validate-template-model", path = "template-model", version = "=1.14.0" }
validation-engine = { package = "cloudformation-validate-validation-engine", path = "validation-engine", version = "=1.14.0" }
cel-engine = { package = "cloudformation-validate-cel-engine", path = "cel-engine", version = "=1.15.0" }
composite-engine = { package = "cloudformation-validate-composite-engine", path = "composite-engine", version = "=1.15.0" }
data-source = { package = "cloudformation-validate-data-source", path = "data-source", version = "=1.15.0", default-features = false }
diagnostics = { package = "cloudformation-validate-diagnostics", path = "diagnostics", version = "=1.15.0" }
guard-translator = { package = "cloudformation-validate-guard-translator", path = "guard-translator", version = "=1.15.0" }
rego-engine = { package = "cloudformation-validate-rego-engine", path = "rego-engine", version = "=1.15.0" }
rules = { package = "cloudformation-validate-rules", path = "rules", version = "=1.15.0" }
schema-validator = { package = "cloudformation-validate-schema-validator", path = "schema-validator", version = "=1.15.0" }
template-model = { package = "cloudformation-validate-template-model", path = "template-model", version = "=1.15.0" }
validation-engine = { package = "cloudformation-validate-validation-engine", path = "validation-engine", version = "=1.15.0" }
uniffi = "=0.32.0"

[profile.dev]
Expand Down
2 changes: 1 addition & 1 deletion src/performance-harness/drift-anchor.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.12.1
1.14.0
54 changes: 7 additions & 47 deletions src/performance-harness/src/measurement.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,23 +225,6 @@ fn collect_template_paths(directory: &Path, output: &mut Vec<PathBuf>) -> Result
Ok(())
}

/// Security fixtures the anchored release cannot measure, so no base launch exists to pair a head launch with: it
/// exhausts memory on the alias expansion and the conditional list items, aborts on alias-expanded nesting, and needs
/// over 20 s per validation of the conditional statements. Releases from `BOUNDED_EXPANSION_RELEASE` on complete them
/// at the fixed baseline; remove the entries when `drift-anchor.txt` reaches that release
/// (`uncomparable_fixtures_exist_until_the_anchor_measures_them`).
const UNCOMPARABLE_SECURITY_FIXTURES: [&str; 4] = [
"independent_conditions_list_items.yaml",
"independent_conditions_statements.yaml",
"yaml_alias_expanded_depth.yaml",
"yaml_alias_expansion.yaml",
];

/// First release that bounds independent-condition and alias expansion, measured against `drift-anchor.txt` by the
/// unit test that retires `UNCOMPARABLE_SECURITY_FIXTURES`.
#[cfg(test)]
const BOUNDED_EXPANSION_RELEASE: (u64, u64, u64) = (1, 14, 0);

fn security_workloads(directory: &Path) -> Result<Vec<Workload>, String> {
let iteration_counts = BTreeMap::from([
("condition_fusion.yaml", 2),
Expand All @@ -256,12 +239,6 @@ fn security_workloads(directory: &Path) -> Result<Vec<Workload>, String> {
]);
let mut templates = Vec::new();
collect_template_paths(directory, &mut templates)?;
templates.retain(|template| {
!template
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| UNCOMPARABLE_SECURITY_FIXTURES.contains(&name))
});
templates.sort();
templates
.into_iter()
Expand Down Expand Up @@ -493,34 +470,17 @@ mod tests {
generate_fixtures(&project_root().join("tmp/performance-harness-unit-test/fixtures")).expect("fixtures");
let workloads = workload_matrix(&fixtures).expect("workloads");
let names: Vec<&str> = workloads.iter().map(|workload| workload.name.as_str()).collect();
assert_eq!(workloads.len(), 22, "{names:?}");
assert_eq!(workloads.len(), 26, "{names:?}");
assert!(names.contains(&"tiny") && names.contains(&"mixed-real"), "{names:?}");
assert!(names.contains(&"security-cross-reference-fanout"), "{names:?}");
assert!(names.contains(&"security-independent-conditions-list-items"), "{names:?}");
assert!(names.contains(&"security-independent-conditions-object-keys"), "{names:?}");
assert!(names.contains(&"security-independent-conditions-properties"), "{names:?}");
assert!(!names.contains(&"security-yaml-alias-expansion"), "{names:?}");
assert!(!names.contains(&"security-yaml-alias-expanded-depth"), "{names:?}");
assert!(names.contains(&"security-independent-conditions-statements"), "{names:?}");
assert!(names.contains(&"security-yaml-alias-expansion"), "{names:?}");
assert!(names.contains(&"security-yaml-alias-expanded-depth"), "{names:?}");
assert!(names.contains(&"security-conditional-scenario-correctness"), "{names:?}");
assert_eq!(ENGINES.len() * workloads.len(), 66);
}

#[test]
fn uncomparable_fixtures_exist_until_the_anchor_measures_them() {
let security = project_root().join("src/resources/security");
for fixture in UNCOMPARABLE_SECURITY_FIXTURES {
assert!(security.join(fixture).is_file(), "{fixture} is not a security fixture");
}
let anchor = fs::read_to_string(project_root().join("src/performance-harness/drift-anchor.txt"))
.expect("drift anchor file");
let release = anchor.trim().trim_end_matches("-beta");
let numbers: Vec<u64> = release.split('.').map(|part| part.parse().expect("release number")).collect();
let anchored = match numbers.as_slice() {
[major, minor, patch] => (*major, *minor, *patch),
other => panic!("expected MAJOR.MINOR.PATCH, found {other:?}"),
};
assert!(
anchored < BOUNDED_EXPANSION_RELEASE,
"release {release} measures every security fixture: remove UNCOMPARABLE_SECURITY_FIXTURES"
);
assert_eq!(ENGINES.len() * workloads.len(), 78);
}

fn workload(name: &str, gate_process_lifecycle: bool) -> Workload {
Expand Down
2 changes: 1 addition & 1 deletion src/resources/expected/version.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.14.0
1.15.0
Loading