diff --git a/.gitignore b/.gitignore index d8d7ec74f..b164ae50d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ *.cdx.json *.folded +*-secret.* +*.tar /flame*.svg /perf.data* diff --git a/common/src/db/func.rs b/common/src/db/func.rs index fc68958bd..3089c0739 100644 --- a/common/src/db/func.rs +++ b/common/src/db/func.rs @@ -77,28 +77,12 @@ impl UpdateDeprecatedAdvisory { } } -/// The function expanding the license replacing all 'LicenseRef-' instances -/// with the actual license they refer to. -pub struct ExpandLicenseExpression; - -impl Iden for ExpandLicenseExpression { - #[allow(clippy::unwrap_used)] - fn unquoted(&self, s: &mut dyn Write) { - write!(s, "expand_license_expression").unwrap() - } -} - -/// The function returns the final license, no matter if it's coming from a CycloneDx of SPDX -/// license data stored in the DB. -pub struct CaseLicenseTextSbomId; - -impl Iden for CaseLicenseTextSbomId { - #[allow(clippy::unwrap_used)] - fn unquoted(&self, s: &mut dyn Write) { - write!(s, "case_license_text_sbom_id").unwrap() - } -} - +// NOTE: This enum is currently unused. The `expand_license_expression_with_mappings` +// PostgreSQL function is invoked via raw SQL in `populate_expanded_license()` due to +// its use of complex PostgreSQL features (composite types, array aggregation over +// `license_mapping`, and complex CTEs). This enum is preserved for potential future +// refactoring to SeaQuery/SeaORM query builders, though such migration may not be +// feasible given the function's complexity and the performance benefits of raw SQL. #[derive(Iden)] pub enum CustomFunc { #[iden = "expand_license_expression_with_mappings"] diff --git a/entity/src/expanded_license.rs b/entity/src/expanded_license.rs new file mode 100644 index 000000000..d0ccbd75a --- /dev/null +++ b/entity/src/expanded_license.rs @@ -0,0 +1,24 @@ +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "expanded_license")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: Uuid, + pub expanded_text: String, + pub text_hash: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::sbom_license_expanded::Entity")] + SbomLicenseExpanded, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::SbomLicenseExpanded.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/entity/src/lib.rs b/entity/src/lib.rs index e39362eef..f14c3ed8c 100644 --- a/entity/src/lib.rs +++ b/entity/src/lib.rs @@ -4,6 +4,7 @@ pub mod base_purl; pub mod cpe; pub mod cvss3; pub mod cvss4; +pub mod expanded_license; pub mod importer; pub mod importer_report; pub mod labels; @@ -22,6 +23,7 @@ pub mod relationship; pub mod sbom; pub mod sbom_external_node; pub mod sbom_file; +pub mod sbom_license_expanded; pub mod sbom_node; pub mod sbom_node_checksum; pub mod sbom_package; diff --git a/entity/src/sbom_license_expanded.rs b/entity/src/sbom_license_expanded.rs new file mode 100644 index 000000000..4b3bb5d88 --- /dev/null +++ b/entity/src/sbom_license_expanded.rs @@ -0,0 +1,53 @@ +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "sbom_license_expanded")] +pub struct Model { + #[sea_orm(primary_key)] + pub sbom_id: Uuid, + #[sea_orm(primary_key)] + pub license_id: Uuid, + pub expanded_license_id: Uuid, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::expanded_license::Entity", + from = "Column::ExpandedLicenseId", + to = "super::expanded_license::Column::Id" + )] + ExpandedLicense, + #[sea_orm( + belongs_to = "super::sbom::Entity", + from = "Column::SbomId", + to = "super::sbom::Column::SbomId" + )] + Sbom, + #[sea_orm( + belongs_to = "super::license::Entity", + from = "Column::LicenseId", + to = "super::license::Column::Id" + )] + License, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::ExpandedLicense.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Sbom.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::License.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/entity/src/sbom_package_license.rs b/entity/src/sbom_package_license.rs index bd9e79dfb..837189212 100644 --- a/entity/src/sbom_package_license.rs +++ b/entity/src/sbom_package_license.rs @@ -27,6 +27,12 @@ pub enum Relation { Package, #[sea_orm(has_one = "super::license::Entity")] License, + #[sea_orm( + belongs_to = "super::sbom_license_expanded::Entity", + from = "(Column::SbomId, Column::LicenseId)", + to = "(super::sbom_license_expanded::Column::SbomId, super::sbom_license_expanded::Column::LicenseId)" + )] + SbomLicenseExpanded, } #[derive( @@ -67,4 +73,10 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::SbomLicenseExpanded.def() + } +} + impl ActiveModelBehavior for ActiveModel {} diff --git a/etc/test-data/spdx/license-ref-overlap.json b/etc/test-data/spdx/license-ref-overlap.json new file mode 100644 index 000000000..7cbae916e --- /dev/null +++ b/etc/test-data/spdx/license-ref-overlap.json @@ -0,0 +1,88 @@ +{ + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "license-ref-overlap-test", + "documentNamespace": "https://example.org/test/license-ref-overlap", + "creationInfo": { + "created": "2024-01-01T00:00:00Z", + "creators": [ + "Tool: test" + ], + "licenseListVersion": "3.19" + }, + "hasExtractedLicensingInfos": [ + { + "licenseId": "LicenseRef-BSD", + "extractedText": "BSD License text", + "name": "BSD License" + }, + { + "licenseId": "LicenseRef-BSD-with-advertising", + "extractedText": "BSD with advertising License text", + "name": "BSD with advertising License" + }, + { + "licenseId": "LicenseRef-GPL", + "extractedText": "GPL License text", + "name": "GPL License" + }, + { + "licenseId": "LicenseRef-GPLv2", + "extractedText": "GPLv2 License text", + "name": "GPLv2 License" + } + ], + "packages": [ + { + "SPDXID": "SPDXRef-Package-bsd-advertising", + "name": "package-bsd-advertising", + "versionInfo": "1.0", + "downloadLocation": "https://example.org/package-bsd-advertising", + "filesAnalyzed": false, + "licenseConcluded": "LicenseRef-BSD-with-advertising", + "licenseDeclared": "LicenseRef-BSD-with-advertising", + "copyrightText": "NOASSERTION", + "supplier": "Organization: Test" + }, + { + "SPDXID": "SPDXRef-Package-bsd-only", + "name": "package-bsd-only", + "versionInfo": "1.0", + "downloadLocation": "https://example.org/package-bsd-only", + "filesAnalyzed": false, + "licenseConcluded": "LicenseRef-BSD", + "licenseDeclared": "LicenseRef-BSD", + "copyrightText": "NOASSERTION", + "supplier": "Organization: Test" + }, + { + "SPDXID": "SPDXRef-Package-gpl-overlap", + "name": "package-gpl-overlap", + "versionInfo": "1.0", + "downloadLocation": "https://example.org/package-gpl-overlap", + "filesAnalyzed": false, + "licenseConcluded": "LicenseRef-GPLv2 OR LicenseRef-GPL", + "licenseDeclared": "LicenseRef-GPLv2 OR LicenseRef-GPL", + "copyrightText": "NOASSERTION", + "supplier": "Organization: Test" + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-Package-bsd-advertising" + }, + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-Package-bsd-only" + }, + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-Package-gpl-overlap" + } + ] +} diff --git a/migration/src/lib.rs b/migration/src/lib.rs index cee3c4d9e..8a2138338 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -33,6 +33,8 @@ mod m0001180_expand_spdx_licenses_with_mappings_function; mod m0001190_optimize_product_advisory_query; mod m0001200_source_document_fk_indexes; mod m0002100_analysis_perf_indexes; +mod m0002110_license_query_performance; +mod m0002120_normalize_expanded_license; pub struct Migrator; @@ -73,6 +75,8 @@ impl MigratorTrait for Migrator { Box::new(m0001190_optimize_product_advisory_query::Migration), Box::new(m0001200_source_document_fk_indexes::Migration), Box::new(m0002100_analysis_perf_indexes::Migration), + Box::new(m0002110_license_query_performance::Migration), + Box::new(m0002120_normalize_expanded_license::Migration), ] } } diff --git a/migration/src/m0002110_license_query_performance.rs b/migration/src/m0002110_license_query_performance.rs new file mode 100644 index 000000000..c7a8b2802 --- /dev/null +++ b/migration/src/m0002110_license_query_performance.rs @@ -0,0 +1,130 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +#[allow(deprecated)] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // HASH index on license(text) for exact-match lookups + manager + .create_index( + Index::create() + .if_not_exists() + .table(License::Table) + .name(Indexes::IdxLicenseTextHash.to_string()) + .col(License::Text) + .index_type(IndexType::Hash) + .to_owned(), + ) + .await?; + + // B-tree composite index on sbom_package_license(license_id, sbom_id) + manager + .create_index( + Index::create() + .if_not_exists() + .table(SbomPackageLicense::Table) + .name(Indexes::IdxSbomPkgLicLicenseSbom.to_string()) + .col(SbomPackageLicense::LicenseId) + .col(SbomPackageLicense::SbomId) + .to_owned(), + ) + .await?; + + // B-tree composite index on licensing_infos(sbom_id, license_id, name) + manager + .create_index( + Index::create() + .if_not_exists() + .table(LicensingInfos::Table) + .name(Indexes::IdxLicensingInfosComposite.to_string()) + .col(LicensingInfos::SbomId) + .col(LicensingInfos::LicenseId) + .col(LicensingInfos::Name) + .to_owned(), + ) + .await?; + + // Replace function with optimized version (early exit for non-LicenseRef texts) + manager + .get_connection() + .execute_unprepared(include_str!("m0002110_license_query_performance/up.sql")) + .await + .map(|_| ())?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Restore original function from m0001180 + manager + .get_connection() + .execute_unprepared(include_str!("m0002110_license_query_performance/down.sql")) + .await + .map(|_| ())?; + + // Drop indexes in reverse order + manager + .drop_index( + Index::drop() + .if_exists() + .table(LicensingInfos::Table) + .name(Indexes::IdxLicensingInfosComposite.to_string()) + .to_owned(), + ) + .await?; + + manager + .drop_index( + Index::drop() + .if_exists() + .table(SbomPackageLicense::Table) + .name(Indexes::IdxSbomPkgLicLicenseSbom.to_string()) + .to_owned(), + ) + .await?; + + manager + .drop_index( + Index::drop() + .if_exists() + .table(License::Table) + .name(Indexes::IdxLicenseTextHash.to_string()) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +#[allow(clippy::enum_variant_names)] +#[derive(DeriveIden)] +enum Indexes { + IdxLicenseTextHash, + IdxSbomPkgLicLicenseSbom, + IdxLicensingInfosComposite, +} + +#[derive(DeriveIden)] +enum SbomPackageLicense { + Table, + LicenseId, + SbomId, +} + +#[derive(DeriveIden)] +enum License { + Table, + Text, +} + +#[derive(DeriveIden)] +enum LicensingInfos { + Table, + SbomId, + LicenseId, + Name, +} diff --git a/migration/src/m0002110_license_query_performance/down.sql b/migration/src/m0002110_license_query_performance/down.sql new file mode 100644 index 000000000..11e6159f3 --- /dev/null +++ b/migration/src/m0002110_license_query_performance/down.sql @@ -0,0 +1,23 @@ +CREATE OR REPLACE FUNCTION expand_license_expression_with_mappings( + license_text TEXT, + license_mappings license_mapping[] +) RETURNS TEXT AS $$ +DECLARE + result_text TEXT := license_text; + mapping license_mapping; +BEGIN + IF license_mappings IS NULL OR array_length(license_mappings, 1) IS NULL OR license_text IS NULL THEN + RETURN license_text; + END IF; + + FOREACH mapping IN ARRAY license_mappings + LOOP + IF result_text !~ 'LicenseRef-' THEN + EXIT; + END IF; + result_text := regexp_replace(result_text, '\m' || mapping.license_id || '\M', mapping.name, 'g'); + END LOOP; + + RETURN result_text; +END; +$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; diff --git a/migration/src/m0002110_license_query_performance/up.sql b/migration/src/m0002110_license_query_performance/up.sql new file mode 100644 index 000000000..8a6946fce --- /dev/null +++ b/migration/src/m0002110_license_query_performance/up.sql @@ -0,0 +1,26 @@ +CREATE OR REPLACE FUNCTION expand_license_expression_with_mappings( + license_text TEXT, + license_mappings license_mapping[] +) RETURNS TEXT AS $$ +DECLARE + result_text TEXT := license_text; + mapping license_mapping; +BEGIN + IF license_text IS NULL + OR POSITION('LicenseRef-' IN license_text) = 0 + OR license_mappings IS NULL + OR array_length(license_mappings, 1) IS NULL THEN + RETURN license_text; + END IF; + + FOREACH mapping IN ARRAY license_mappings + LOOP + IF POSITION('LicenseRef-' IN result_text) = 0 THEN + EXIT; + END IF; + result_text := regexp_replace(result_text, '\m' || mapping.license_id || '(?![a-zA-Z0-9.\-])', mapping.name, 'g'); + END LOOP; + + RETURN result_text; +END; +$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; diff --git a/migration/src/m0002120_normalize_expanded_license.rs b/migration/src/m0002120_normalize_expanded_license.rs new file mode 100644 index 000000000..9cb513c38 --- /dev/null +++ b/migration/src/m0002120_normalize_expanded_license.rs @@ -0,0 +1,25 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared(include_str!("m0002120_normalize_expanded_license/up.sql")) + .await + .map(|_| ())?; + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared(include_str!("m0002120_normalize_expanded_license/down.sql")) + .await + .map(|_| ())?; + Ok(()) + } +} diff --git a/migration/src/m0002120_normalize_expanded_license/down.sql b/migration/src/m0002120_normalize_expanded_license/down.sql new file mode 100644 index 000000000..3202c1077 --- /dev/null +++ b/migration/src/m0002120_normalize_expanded_license/down.sql @@ -0,0 +1,38 @@ +-- Drop new tables +DROP TABLE IF EXISTS sbom_license_expanded; +DROP TABLE IF EXISTS expanded_license; + +-- Restore old functions for backward compatibility + +-- expand_license_expression (from m0001160) +CREATE OR REPLACE FUNCTION expand_license_expression( + license_text TEXT, + sbom_id_param UUID +) RETURNS TEXT AS $$ +DECLARE + result_text TEXT := license_text; + license_mapping RECORD; +BEGIN + FOR license_mapping IN + SELECT license_id, name + FROM licensing_infos + WHERE sbom_id = sbom_id_param + LOOP + IF result_text !~ 'LicenseRef-' THEN + EXIT; + END IF; + result_text := regexp_replace(result_text, '\m' || license_mapping.license_id || '\M', license_mapping.name, 'g'); + END LOOP; + RETURN result_text; +END; +$$ LANGUAGE plpgsql STABLE; + +-- case_license_text_sbom_id (from m0001150) +CREATE OR REPLACE FUNCTION case_license_text_sbom_id( + license_text TEXT, + sbom_id_param UUID +) RETURNS TEXT AS $$ +BEGIN + RETURN expand_license_expression(license_text, sbom_id_param); +END; +$$ LANGUAGE plpgsql STABLE; diff --git a/migration/src/m0002120_normalize_expanded_license/up.sql b/migration/src/m0002120_normalize_expanded_license/up.sql new file mode 100644 index 000000000..a92f7bffe --- /dev/null +++ b/migration/src/m0002120_normalize_expanded_license/up.sql @@ -0,0 +1,88 @@ +-- Create dictionary table for unique expanded license texts +CREATE TABLE IF NOT EXISTS expanded_license ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + expanded_text TEXT NOT NULL, + text_hash TEXT GENERATED ALWAYS AS (md5(expanded_text)) STORED +); + +-- Unique constraint on the generated hash column for deduplication +-- (handles long texts >2.7KB that exceed B-tree limits) +CREATE UNIQUE INDEX IF NOT EXISTS idx_expanded_license_text_hash +ON expanded_license (text_hash); + +-- Create junction table mapping (sbom_id, license_id) → expanded_license_id +CREATE TABLE IF NOT EXISTS sbom_license_expanded ( + sbom_id UUID NOT NULL, + license_id UUID NOT NULL, + expanded_license_id UUID NOT NULL, + PRIMARY KEY (sbom_id, license_id), + FOREIGN KEY (sbom_id) REFERENCES sbom(sbom_id) ON DELETE CASCADE, + FOREIGN KEY (license_id) REFERENCES license(id) ON DELETE CASCADE, + FOREIGN KEY (expanded_license_id) REFERENCES expanded_license(id) ON DELETE CASCADE +); + +-- Index for reverse lookups (expanded_license_id → sbom_license_expanded) +CREATE INDEX IF NOT EXISTS idx_sle_expanded_license_id +ON sbom_license_expanded (expanded_license_id); + +-- Backfill Step 1: Insert unique expanded texts into dictionary +-- Pre-deduplicate by (text, sbom_id) to avoid millions of redundant function calls +INSERT INTO expanded_license (expanded_text) +SELECT DISTINCT expand_license_expression_with_mappings( + uls.text, + COALESCE(lim.license_mapping, ARRAY[]::license_mapping[]) +) +FROM ( + SELECT DISTINCT l.text, spl.sbom_id + FROM sbom_package_license spl + JOIN license l ON l.id = spl.license_id +) uls +LEFT JOIN ( + SELECT array_agg(ROW(license_id, name)::license_mapping) AS license_mapping, sbom_id + FROM licensing_infos + GROUP BY sbom_id +) lim ON lim.sbom_id = uls.sbom_id +-- Filter at SBOM level (not per-license-id) since SBOMs are immutable once ingested. +-- If ANY license from an SBOM has been backfilled, all have been backfilled. +WHERE NOT EXISTS ( + SELECT 1 FROM sbom_license_expanded sle + WHERE sle.sbom_id = uls.sbom_id +) +ON CONFLICT (text_hash) DO NOTHING; + +-- Backfill Step 2: Insert junction rows +-- Use a CTE (license_expansions) to call expand_license_expression_with_mappings() only once per +-- (sbom_id, license_id) pair; the result is then joined against the already-populated +-- expanded_license dictionary by md5 hash, avoiding a second expensive function call. +WITH license_expansions AS ( + SELECT DISTINCT + spl.sbom_id, + spl.license_id, + expand_license_expression_with_mappings( + l.text, + COALESCE(lim.license_mapping, ARRAY[]::license_mapping[]) + ) AS expanded_text + FROM sbom_package_license spl + JOIN license l ON l.id = spl.license_id + LEFT JOIN ( + SELECT array_agg(ROW(license_id, name)::license_mapping) AS license_mapping, sbom_id + FROM licensing_infos + GROUP BY sbom_id + ) lim ON lim.sbom_id = spl.sbom_id + WHERE NOT EXISTS ( + SELECT 1 FROM sbom_license_expanded sle + WHERE sle.sbom_id = spl.sbom_id AND sle.license_id = spl.license_id + ) +) +INSERT INTO sbom_license_expanded (sbom_id, license_id, expanded_license_id) +SELECT ne.sbom_id, ne.license_id, el.id +FROM license_expansions ne +JOIN expanded_license el ON el.text_hash = md5(ne.expanded_text) +ON CONFLICT (sbom_id, license_id) DO UPDATE +SET expanded_license_id = EXCLUDED.expanded_license_id; + +-- Drop old SQL functions (no longer needed after backfill) +DROP FUNCTION IF EXISTS case_license_text_sbom_id(TEXT, UUID); +DROP FUNCTION IF EXISTS expand_license_expression(TEXT, UUID); + +-- Keep expand_license_expression_with_mappings() for ingestion-time use diff --git a/modules/analysis/src/lib.rs b/modules/analysis/src/lib.rs index 01a2dfedf..c9b3a533f 100644 --- a/modules/analysis/src/lib.rs +++ b/modules/analysis/src/lib.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "512"] + pub mod config; pub mod endpoints; pub mod error; diff --git a/modules/fundamental/src/common/license_filtering.rs b/modules/fundamental/src/common/license_filtering.rs index 5b2bca47b..26140a066 100644 --- a/modules/fundamental/src/common/license_filtering.rs +++ b/modules/fundamental/src/common/license_filtering.rs @@ -1,301 +1,21 @@ -use crate::Error; -use sea_orm::{ - ColumnTrait, EntityTrait, QueryFilter, QuerySelect, QueryTrait, RelationTrait, Select, -}; -use sea_query::{ - Alias, ColumnType, CommonTableExpression, Condition, Expr, Func, JoinType, PgFunc, - SelectStatement, SimpleExpr, UnionType, WithClause, extension::postgres::PgExpr, -}; -use trustify_common::db::{ - CaseLicenseTextSbomId, CustomFunc, ExpandLicenseExpression, - query::{Columns, Filtering, IntoColumns, Query, q}, -}; -use trustify_entity::{ - license, licensing_infos, sbom_package, sbom_package_license, sbom_package_purl_ref, -}; +use sea_orm::IntoSimpleExpr; +use sea_query::{Expr, Func, SimpleExpr}; +use trustify_entity::{expanded_license, license}; +// License field constant used in query filtering pub const LICENSE: &str = "license"; -/// Builds a CycloneDX license query using direct text matching on license fields +/// Creates a COALESCE expression that prefers expanded license text over raw license text. /// -/// # Arguments -/// * `license_query` - The license query to filter by -/// * `base_query` - The base query to apply license filtering to -fn build_cyclonedx_license_query( - license_query: Query, - base_query: Select, -) -> Result -where - E: EntityTrait, -{ - Ok(base_query - .filtering_with( - license_query, - license::Entity - .columns() - .translator(|field, operator, value| match field { - LICENSE => Some(format!("text{operator}{value}")), - _ => None, - }), - )? - .into_query()) -} - -/// Builds an SPDX license query using expand_license_expression() for LicenseRef resolution -/// -/// # Arguments -/// * `license_query` - The license query to filter by -/// * `base_query` - The base query to apply license filtering to -fn build_spdx_license_query( - license_query: Query, - base_query: Select, -) -> Result -where - E: EntityTrait, -{ - const EXPANDED_LICENSE: &str = "expanded_license"; - Ok(base_query - .filtering_with( - license_query, - Columns::default() - .add_expr( - EXPANDED_LICENSE, - SimpleExpr::FunctionCall( - Func::cust(ExpandLicenseExpression) - .arg(Expr::col(license::Column::Text)) - .arg(Expr::col(( - sbom_package_license::Entity, - sbom_package_license::Column::SbomId, - ))), - ), - ColumnType::Text, - ) - .translator(|field, operator, value| match field { - LICENSE => Some(format!("{EXPANDED_LICENSE}{operator}{value}")), - _ => None, - }), - )? - .filter(Expr::col(license::Column::Text).ilike("%LicenseRef-%")) - .into_query()) -} - -/// Creates a base query for PURL license filtering (targeting qualified_purl_id) -pub fn create_purl_license_filtering_base_query() -> Select { - sbom_package_purl_ref::Entity::find() - .select_only() - .column(sbom_package_purl_ref::Column::QualifiedPurlId) - .join( - JoinType::Join, - sbom_package_purl_ref::Relation::Package.def(), - ) - .join(JoinType::Join, sbom_package::Relation::PackageLicense.def()) - .join( - JoinType::Join, - sbom_package_license::Relation::License.def(), - ) -} - -/// Creates a base query for SBOM license filtering (targeting sbom_id) -pub fn create_sbom_license_filtering_base_query() -> Select { - sbom_package_license::Entity::find() - .select_only() - .column(sbom_package_license::Column::SbomId) - .join( - JoinType::Join, - sbom_package_license::Relation::License.def(), - ) -} - -/// Creates a base query for SBOM package license filtering (targeting packages within a specific SBOM) -pub fn create_sbom_package_license_filtering_base_query( - sbom_id: sea_orm::prelude::Uuid, -) -> Select { - sbom_package::Entity::find() - .filter(sbom_package::Column::SbomId.eq(sbom_id)) - .select_only() - .column(sbom_package::Column::NodeId) - .join(JoinType::Join, sbom_package::Relation::PackageLicense.def()) - .join( - JoinType::Join, - sbom_package_license::Relation::License.def(), - ) -} - -/// Applies license filtering to a query using a two-phase SPDX/CycloneDX approach -/// -/// This function encapsulates the complete license filtering pattern used by both -/// PURL and SBOM services, eliminating code duplication. -/// -/// # Arguments -/// * `main_query` - The main query to apply license filtering to -/// * `search_query` - The full search query that may contain license constraints -/// * `base_query_fn` - Function that creates the base query for license filtering -/// * `target_column` - The column to use in the subquery (e.g., qualified_purl::Column::Id or sbom::Column::SbomId) -/// -/// # Returns -/// The modified main query with license filtering applied (if license constraints exist) -pub fn apply_license_filtering( - main_query: Select, - search_query: &Query, - base_query_fn: F, - target_column: C, -) -> Result, Error> -where - E: EntityTrait, - BE: EntityTrait, - F: Fn() -> Select, - C: ColumnTrait, -{ - // since different fields conditions in input query are AND'd when translating them - // into DB query, if the `license` field is in the input query then qualified_purl - // that will match the input query criteria must be among the one satisfying - // the license values requested in the input query itself. - if let Some(license_query) = search_query - .get_constraint_for_field(LICENSE) - .map(|constraint| q(&format!("{constraint}"))) - { - let license_filtering_base_query = base_query_fn(); - let mut select_from_spdx = - build_spdx_license_query(license_query.clone(), license_filtering_base_query.clone())?; - let select_from_cyclonedx = - build_cyclonedx_license_query(license_query, license_filtering_base_query)?; - - // Filters using a two-phase approach: - // 1. SPDX documents: Uses expand_license_expression() for LicenseRef resolution - // 2. CycloneDX documents: Direct text matching on license field - // The results are UNIONed and used to filter the main query. - let select_filtering_by_license = - select_from_spdx.union(UnionType::Distinct, select_from_cyclonedx); - - Ok(main_query.filter( - Condition::all().add(target_column.in_subquery(select_filtering_by_license.clone())), +/// Returns: `COALESCE(expanded_license.expanded_text, license.text)` +pub fn license_text_coalesce() -> SimpleExpr { + Func::coalesce([ + Expr::col(( + expanded_license::Entity, + expanded_license::Column::ExpandedText, )) - } else { - // No license filtering needed, return the query unchanged - Ok(main_query) - } -} - -/// Returns the case_license_text_sbom_id() PLSQL function that conditionally applies expand_license_expression() for SPDX LicenseRefs -/// -/// This function generates a SQL CASE expression that: -/// - Returns the expanded license expression when the license text contains 'LicenseRef-' (SPDX format) -/// - Returns the original license text for all other cases (including CycloneDX) -/// -/// This allows unified handling of both SPDX and CycloneDX licenses in a single query. -pub fn get_case_license_text_sbom_id() -> SimpleExpr { - SimpleExpr::FunctionCall( - Func::cust(CaseLicenseTextSbomId) - .arg(Expr::col((license::Entity, license::Column::Text))) - .arg(Expr::col(( - sbom_package_license::Entity, - sbom_package_license::Column::SbomId, - ))), - ) -} - -/// Builds a WithClause containing the three CTEs required for SPDX license filtering -/// -/// This function creates the Common Table Expressions (CTEs) needed to handle SPDX license -/// expression expansion with LicenseRef mappings: -/// -/// 1. `licensing_infos_mappings` - Aggregates license ID/name mappings per SBOM -/// 2. `unique_license_sbom` - Deduplicates license text by (license_text, sbom_id) -/// 3. `expanded` - Applies expand_license_expression_with_mappings() to resolve LicenseRefs -/// -/// # Returns -/// A WithClause containing all three CTEs, ready to be attached to a query via `.with()` -/// -/// # Example Usage -/// ```rust -/// use sea_orm::{EntityTrait, QueryTrait}; -/// use trustify_entity::sbom; -/// use trustify_module_fundamental::common::license_filtering::build_license_filtering_with_clause; -/// -/// let with_clause = build_license_filtering_with_clause(); -/// let my_select_query = sbom::Entity::find(); -/// let query = my_select_query.into_query().with(with_clause); -/// ``` -pub fn build_license_filtering_with_clause() -> WithClause { - // licensing_infos_mappings CTE - let licensing_infos_mappings_query = licensing_infos::Entity::find() - .select_only() - .expr_as( - PgFunc::array_agg( - Expr::cust_with_exprs( - "ROW($1, $2)", - [ - Expr::col(licensing_infos::Column::LicenseId).into(), - Expr::col(licensing_infos::Column::Name).into(), - ], - ) - .cast_as("license_mapping"), - ), - "license_mapping", - ) - .column(licensing_infos::Column::SbomId) - .group_by(licensing_infos::Column::SbomId); - - let licensing_infos_mappings_cte = CommonTableExpression::new() - .query(licensing_infos_mappings_query.into_query()) - .table_name(Alias::new("licensing_infos_mappings")) - .to_owned(); - - // unique_license_sbom CTE - let unique_license_sbom_query = sbom_package_license::Entity::find() - .distinct() - .select_only() - .column(license::Column::Text) - .expr(Expr::col(( - sbom_package_license::Entity, - sbom_package_license::Column::SbomId, - ))) - .column_as(license::Column::Id, "license_id") - .join( - JoinType::Join, - sbom_package_license::Relation::License.def(), - ); - - let unique_license_sbom_cte = CommonTableExpression::new() - .query(unique_license_sbom_query.into_query()) - .table_name(Alias::new("unique_license_sbom")) - .to_owned(); - - // expanded CTE - let expanded_query = sea_query::Query::select() - .column((Alias::new("unique_license_sbom"), Alias::new("sbom_id"))) - .column((Alias::new("unique_license_sbom"), Alias::new("license_id"))) - .expr_as( - Func::cust(CustomFunc::ExpandLicenseExpressionWithMappings).args([ - Expr::col((Alias::new("unique_license_sbom"), Alias::new("text"))).into(), - Expr::col(( - Alias::new("licensing_infos_mappings"), - Alias::new("license_mapping"), - )) - .into(), - ]), - Alias::new("expanded_text"), - ) - .from(Alias::new("unique_license_sbom")) - .join( - JoinType::LeftJoin, - Alias::new("licensing_infos_mappings"), - Expr::col((Alias::new("unique_license_sbom"), Alias::new("sbom_id"))).equals(( - Alias::new("licensing_infos_mappings"), - Alias::new("sbom_id"), - )), - ) - .to_owned(); - - let expanded_cte = CommonTableExpression::new() - .query::(expanded_query) - .table_name(Alias::new("expanded")) - .to_owned(); - - // Combine all CTEs into a WithClause - WithClause::new() - .cte(licensing_infos_mappings_cte) - .cte(unique_license_sbom_cte) - .cte(expanded_cte) - .to_owned() + .into_simple_expr(), + Expr::col((license::Entity, license::Column::Text)).into_simple_expr(), + ]) + .into() } diff --git a/modules/fundamental/src/license/endpoints/test.rs b/modules/fundamental/src/license/endpoints/test.rs index f15cfce10..5237887d9 100644 --- a/modules/fundamental/src/license/endpoints/test.rs +++ b/modules/fundamental/src/license/endpoints/test.rs @@ -314,6 +314,73 @@ async fn list_licenses_with_pagination(ctx: &TrustifyContext) -> Result<(), anyh Ok(()) } +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn list_licenses_no_partial_license_ref_match( + ctx: &TrustifyContext, +) -> Result<(), anyhow::Error> { + let app = caller(ctx).await?; + + // Ingest an SPDX SBOM with overlapping LicenseRef- identifiers: + // LicenseRef-BSD ("BSD License") vs LicenseRef-BSD-with-advertising ("BSD with advertising License") + // LicenseRef-GPL ("GPL License") vs LicenseRef-GPLv2 ("GPLv2 License") + ctx.ingest_document("spdx/license-ref-overlap.json").await?; + + let uri = "/api/v2/license?limit=1000"; + let request = TestRequest::get().uri(uri).to_request(); + let response: PaginatedResults = app.call_and_read_body_json(request).await; + + let license_names: Vec = response.items.iter().map(|l| l.license.clone()).collect(); + + // The bug: LicenseRef-BSD matches within LicenseRef-BSD-with-advertising, + // producing "BSD License-with-advertising" instead of "BSD with advertising License" + assert!( + !license_names + .iter() + .any(|l| l.contains("BSD License-with-advertising")), + "Partial LicenseRef- match detected: 'BSD License-with-advertising' should not exist" + ); + + // Correct expansion of the full LicenseRef-BSD-with-advertising + assert!( + license_names + .iter() + .any(|l| l.contains("BSD with advertising License")), + "Expected 'BSD with advertising License' from LicenseRef-BSD-with-advertising expansion" + ); + + // Short ref LicenseRef-BSD still works on its own + assert!( + license_names.iter().any(|l| l == "BSD License"), + "Expected 'BSD License' from LicenseRef-BSD expansion" + ); + + // Both overlapping GPL refs expand correctly + assert!( + license_names + .iter() + .any(|l| l == "GPLv2 License OR GPL License"), + "Expected 'GPLv2 License OR GPL License' from overlapping GPL LicenseRef expansion, found: {:?}", + license_names + .iter() + .filter(|l| l.contains("GPL")) + .collect::>() + ); + + // No raw LicenseRef- values should remain + let license_ref_found: Vec<&String> = license_names + .iter() + .filter(|l| l.contains("LicenseRef-")) + .collect(); + assert!( + license_ref_found.is_empty(), + "No raw 'LicenseRef-' should remain but found: {:?}", + license_ref_found + ); + + Ok(()) +} + #[test_context(TrustifyContext)] #[test(actix_web::test)] async fn list_licenses_sorting(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { diff --git a/modules/fundamental/src/license/service/mod.rs b/modules/fundamental/src/license/service/mod.rs index d0e7277a8..16859e602 100644 --- a/modules/fundamental/src/license/service/mod.rs +++ b/modules/fundamental/src/license/service/mod.rs @@ -1,8 +1,8 @@ use crate::{ Error, common::{ - LicenseRefMapping, license_filtering, - license_filtering::{LICENSE, build_license_filtering_with_clause}, + LicenseRefMapping, + license_filtering::{LICENSE, license_text_coalesce}, }, license::model::{ SpdxLicenseDetails, SpdxLicenseSummary, @@ -13,26 +13,28 @@ use crate::{ }; use sea_orm::{ ColumnTrait, ConnectionTrait, DatabaseBackend, EntityTrait, FromQueryResult, QueryFilter, - QuerySelect, QueryTrait, RelationTrait, Statement, + QueryOrder, QuerySelect, QueryTrait, RelationTrait, Statement, }; use sea_query::{ - Alias, ColumnType, Condition, Expr, JoinType, Order::Asc, PostgresQueryBuilder, UnionType, - query, + Asterisk, Condition, Expr, Func, JoinType, PostgresQueryBuilder, SimpleExpr, UnionType, }; use serde::{Deserialize, Serialize}; use trustify_common::{ - db::query::{Columns, Filtering, Query}, + db::query::{Columns, Filtering, IntoColumns, Query, q}, id::{Id, TrySelectForId}, model::{Paginated, PaginatedResults}, }; use trustify_entity::{ - license, licensing_infos, qualified_purl, sbom, sbom_node, sbom_package, sbom_package_cpe_ref, - sbom_package_license, sbom_package_purl_ref, + expanded_license, license, licensing_infos, qualified_purl, sbom, sbom_license_expanded, + sbom_node, sbom_package, sbom_package_cpe_ref, sbom_package_license, sbom_package_purl_ref, }; use utoipa::ToSchema; pub mod license_export; +#[cfg(test)] +mod test; + pub struct LicenseService {} pub struct LicenseExportResult { @@ -235,36 +237,35 @@ impl LicenseService { .one(connection) .await?; - const EXPANDED_LICENSE: &str = "expanded_license"; - const LICENSE_NAME: &str = "license_name"; match sbom { Some(sbom) => { - let expand_license_expression = sbom_package_license::Entity::find() + // Build the COALESCE expression once: prefer pre-expanded text, fall back to raw + // license text. Reused for both SELECT columns and ORDER BY to avoid repetition. + let coalesce_expr = license_text_coalesce(); + + let licenses = sbom_package_license::Entity::find() .select_only() .distinct() - .column_as( - license_filtering::get_case_license_text_sbom_id(), - EXPANDED_LICENSE, + .column_as(coalesce_expr.clone(), "license_name") + .column_as(coalesce_expr.clone(), "license_id") + .filter(sbom_package_license::Column::SbomId.eq(sbom.sbom_id)) + .join( + JoinType::LeftJoin, + sbom_package_license::Relation::SbomLicenseExpanded.def(), + ) + .join( + JoinType::LeftJoin, + sbom_license_expanded::Relation::ExpandedLicense.def(), ) .join( - JoinType::Join, + JoinType::LeftJoin, sbom_package_license::Relation::License.def(), ) - .filter(sbom_package_license::Column::SbomId.eq(sbom.sbom_id)); - let (sql, values) = query::Query::select() - // reported twice to keep compatibility with LicenseRefMapping currently - // exposed in the involved endpoint. - .expr_as(Expr::col(Alias::new(EXPANDED_LICENSE)), LICENSE_NAME) - .expr_as(Expr::col(Alias::new(EXPANDED_LICENSE)), "license_id") - .from_subquery(expand_license_expression.into_query(), "expanded_licenses") - .order_by(LICENSE_NAME, Asc) - .build(PostgresQueryBuilder); - let result: Vec = LicenseRefMapping::find_by_statement( - Statement::from_sql_and_values(connection.get_database_backend(), sql, values), - ) - .all(connection) - .await?; - Ok(Some(result)) + .order_by_asc(coalesce_expr) + .into_model::() + .all(connection) + .await?; + Ok(Some(licenses)) } None => Ok(None), } @@ -276,120 +277,104 @@ impl LicenseService { paginated: Paginated, connection: &C, ) -> Result, Error> { - // Build the CTEs for license filtering - let with_clause = build_license_filtering_with_clause(); - const LICENSE_TEXT: &str = "text"; - const EXPANDED_LICENSE: &str = "expanded_text"; - // Let's build a Select in order to, further down, use filtering_with function - let mut base_query = sbom::Entity::find() - .distinct() + + // Build query for SPDX licenses (from expanded_license dictionary) + let mut spdx_query = expanded_license::Entity::find() .select_only() - .expr_as(Expr::col(EXPANDED_LICENSE), LICENSE_TEXT); - // Basically the sorting and the querying can not be applied at the same time because - // they work against different target columns that causes issue - // when a full-text search query is executed because it would be applied also to - // "sort" column in a phase when it won't be available yet in the query. - let Query { ref q, ref sort } = search; - // add query condition - if !q.is_empty() { - base_query = base_query.filtering_with( - trustify_common::db::query::q(&q.to_string()), - Columns::default() - .add_column(EXPANDED_LICENSE, ColumnType::Text) - .translator(|field, operator, value| match (field, operator) { - (LICENSE, _) => Some(format!("{EXPANDED_LICENSE}{operator}{value}")), - _ => None, - }), - )?; - } - // add sorting condition - if !sort.is_empty() { - base_query = base_query.filtering_with( - trustify_common::db::query::q("").sort(sort), - Columns::default() - .add_column(LICENSE_TEXT, ColumnType::Text) - .translator(|field, operator, _value| match (field, operator) { - (LICENSE, "asc" | "desc") => Some(format!("{}:{operator}", LICENSE_TEXT)), - _ => None, - }), - )?; - } - let mut statement = base_query.into_query().to_owned(); - let mut license_texts = statement.join( - JoinType::Join, - Alias::new("expanded"), - Condition::all().add( - Expr::col((sbom::Entity, sbom::Column::SbomId)) - .equals((Alias::new("expanded"), Alias::new("sbom_id"))), - ), - ); - - let default_licenses_with_no_sboms = license::Entity::find() .distinct() + .column_as(expanded_license::Column::ExpandedText, LICENSE_TEXT); + + // Build query for licenses not yet linked to any SBOM: includes both + // (a) pre-loaded SPDX dictionary entries with no SBOM connection yet, AND + // (b) licenses from older SBOMs ingested before license expansion was implemented. + // Use NOT EXISTS instead of LEFT JOIN + IS NULL to find licenses without SBOMs. + // On large tables, LEFT JOIN scans all rows while NOT EXISTS + // uses a Nested Loop Anti Join with index-only scan. + let exists_subquery = sea_query::Query::select() + .expr(Expr::val(1)) + .from(sbom_license_expanded::Entity) + .and_where( + Expr::col(( + sbom_license_expanded::Entity, + sbom_license_expanded::Column::LicenseId, + )) + .equals((license::Entity, license::Column::Id)), + ) + .to_owned(); + + let mut non_sbom_query = license::Entity::find() .select_only() - .column(license::Column::Text) - .join(JoinType::LeftJoin, license::Relation::PackageLicense.def()) - .filter(sbom_package_license::Column::SbomId.is_null()) + .distinct() + .column_as(license::Column::Text, LICENSE_TEXT) + .filter(Expr::exists(exists_subquery).not()); + + // Apply filtering to both queries (without sorting - that's applied to the UNION result) + let filter_only = Query { + q: search.q.clone(), + sort: String::new(), // Don't sort individual queries before UNION + }; + + let spdx_columns = + expanded_license::Entity + .columns() + .translator(|field, operator, value| match field { + LICENSE => Some(format!("expanded_text{operator}{value}")), + _ => None, + }); + + let non_sbom_columns = license::Entity + .columns() + .translator(|field, operator, value| match field { + LICENSE => Some(format!("text{operator}{value}")), + _ => None, + }); + + spdx_query = spdx_query.filtering_with(filter_only.clone(), spdx_columns)?; + non_sbom_query = non_sbom_query.filtering_with(filter_only, non_sbom_columns)?; + + // Union the two queries + QueryTrait::query(&mut spdx_query).union(UnionType::Distinct, non_sbom_query.into_query()); + // Add an expression for the license field and use it as the default sort + let expr = SimpleExpr::Custom(LICENSE_TEXT.into()); + spdx_query = spdx_query .filtering_with( - search.clone(), - Columns::default() - .add_column(license::Column::Text, ColumnType::Text) - .translator(|field, operator, value| match (field, operator) { - (LICENSE, "asc" | "desc") => Some(format!("{}:{operator}", LICENSE_TEXT)), - (LICENSE, _) => Some(format!("{}{operator}{value}", LICENSE_TEXT)), - _ => None, - }), + q("").sort(&search.sort), + Columns::default().add_expr("license", expr.clone(), sea_orm::ColumnType::Text), )? - .into_query() - .to_owned(); + .order_by_asc(expr); - license_texts = - license_texts.union(UnionType::Distinct, default_licenses_with_no_sboms.clone()); + let mut union_query = spdx_query.into_query(); - let license_texts_count = sea_query::Query::select() - .expr_as(Expr::cust("count(*)"), "num_items") - .from_subquery(license_texts.clone(), "subquery") + // Count total results + let count_query = sea_query::Query::select() + .expr_as(Func::count(Expr::col(Asterisk)), "num_items") + .from_subquery(union_query.clone(), "subquery") .to_owned(); - let (sql_count, values) = license_texts_count - .clone() - .with(with_clause.clone()) - .build(PostgresQueryBuilder); - // the standard approach for counting can not be used because it doesn't work with CTE - // since the generated query starts with: - // SELECT COUNT(*) AS num_items FROM (SELECT licensing_infos_mappings" - // which is not SQL syntactically correct - // let selector_raw = LicenseText::find_by_statement(Statement::from_sql_and_values( - // DatabaseBackend::Postgres, - // sql_count.clone(), - // values.clone(), - // )); - // let total = selector_raw.count(connection).await?; - let selector_raw = Statement::from_sql_and_values( - DatabaseBackend::Postgres, - sql_count.clone(), - values.clone(), - ); #[derive(Debug, Default, Clone, Serialize, Deserialize, ToSchema, FromQueryResult)] struct Count { - // It should be u64 but PostgreSQL doesn't support it - // https://www.sea-ql.org/SeaORM/docs/1.1.x/generate-entity/column-types/#type-mappings num_items: i64, } - let total = Count::find_by_statement(selector_raw) - .one(connection) - .await? - .unwrap_or(Count { num_items: 0 }) - .num_items as u64; - let select_paginated = license_texts + let (sql_count, values) = count_query.build(PostgresQueryBuilder); + let total = Count::find_by_statement(Statement::from_sql_and_values( + DatabaseBackend::Postgres, + sql_count, + values, + )) + .one(connection) + .await? + .unwrap_or(Count { num_items: 0 }) + .num_items as u64; + + // Apply pagination + union_query = union_query .offset(paginated.offset) .limit(paginated.limit) .to_owned(); - let (sql, values) = select_paginated - .with(with_clause) - .build(PostgresQueryBuilder); + + let (sql, values) = union_query.build(PostgresQueryBuilder); let items = LicenseText::find_by_statement(Statement::from_sql_and_values( DatabaseBackend::Postgres, sql, @@ -397,6 +382,7 @@ impl LicenseService { )) .all(connection) .await?; + Ok(PaginatedResults { total, items }) } } diff --git a/modules/fundamental/src/license/service/test.rs b/modules/fundamental/src/license/service/test.rs new file mode 100644 index 000000000..5d660a1a4 --- /dev/null +++ b/modules/fundamental/src/license/service/test.rs @@ -0,0 +1,537 @@ +use crate::license::service::LicenseService; +use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter}; +use test_context::test_context; +use test_log::test; +use trustify_common::{db::query::Query, id::Id, model::Paginated}; +use trustify_entity::{expanded_license, sbom_license_expanded}; +use trustify_test_context::TrustifyContext; + +/// RED-GREEN-REFACTOR: Test licenses() UNION query +/// Verifies: expanded_license.expanded_text UNION license.text for CycloneDX +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_licenses_union_spdx_and_cyclonedx( + ctx: &TrustifyContext, +) -> Result<(), anyhow::Error> { + let service = LicenseService::new(); + + // RED: Before ingestion + let result_before = service + .licenses(Query::default(), Paginated::default(), &ctx.db) + .await?; + let baseline_count = result_before.total; + + // GREEN: Ingest SPDX (uses expanded_license) and CycloneDX (uses license.text directly) + ctx.ingest_documents([ + "spdx/OCP-TOOLS-4.11-RHEL-8.json", + "zookeeper-3.9.2-cyclonedx.json", + ]) + .await?; + + // REFACTOR: Verify UNION includes both sources + let result_after = service + .licenses( + Query::default(), + Paginated { + offset: 0, + limit: 1000, + }, + &ctx.db, + ) + .await?; + + assert!( + result_after.total > baseline_count, + "Should have more licenses after ingestion" + ); + + // Verify expanded_license table has SPDX data + let expanded_count = expanded_license::Entity::find().count(&ctx.db).await?; + assert!(expanded_count > 0, "expanded_license should have SPDX data"); + + // Verify no raw LicenseRef- in results (SPDX should be expanded) + let has_license_ref = result_after + .items + .iter() + .any(|l| l.license.contains("LicenseRef-")); + assert!( + !has_license_ref, + "Results should not contain raw LicenseRef-" + ); + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test get_all_license_info() COALESCE logic +/// Verifies: COALESCE(expanded_license.expanded_text, license.text) +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_get_all_license_info_coalesce(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let service = LicenseService::new(); + + // RED: Ingest SPDX with LicenseRef + let result = ctx + .ingest_document("spdx/OCP-TOOLS-4.11-RHEL-8.json") + .await?; + let sbom_id = result + .id + .try_as_uid() + .ok_or_else(|| anyhow::anyhow!("Expected UUID ID"))?; + + // GREEN: Get license info + let info = service + .get_all_license_info(Id::Uuid(sbom_id), &ctx.db) + .await? + .expect("Should have license info"); + + // REFACTOR: Verify data uses COALESCE - no LicenseRef-, all expanded + for mapping in &info { + assert!( + !mapping.license_name.contains("LicenseRef-"), + "license_name should be expanded: {}", + mapping.license_name + ); + assert_eq!( + mapping.license_id, mapping.license_name, + "Both fields use same COALESCE" + ); + } + + // Verify junction table was consulted + let junction_count = sbom_license_expanded::Entity::find() + .filter(sbom_license_expanded::Column::SbomId.eq(sbom_id)) + .count(&ctx.db) + .await?; + assert!(junction_count > 0, "Junction table should have entries"); + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test junction table (sbom_id, license_id) → expanded_license_id mapping +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_junction_table_mapping_integrity(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + // RED: Empty junction + assert_eq!( + sbom_license_expanded::Entity::find().count(&ctx.db).await?, + 0 + ); + + // GREEN: Ingest + let result = ctx + .ingest_document("spdx/OCP-TOOLS-4.11-RHEL-8.json") + .await?; + let sbom_id = result + .id + .try_as_uid() + .ok_or_else(|| anyhow::anyhow!("Expected UUID ID"))?; + + // REFACTOR: Verify every junction entry references valid expanded_license + let entries = sbom_license_expanded::Entity::find() + .filter(sbom_license_expanded::Column::SbomId.eq(sbom_id)) + .all(&ctx.db) + .await?; + + assert!(!entries.is_empty(), "Should have junction entries"); + + for entry in entries { + let expanded = expanded_license::Entity::find_by_id(entry.expanded_license_id) + .one(&ctx.db) + .await?; + assert!( + expanded.is_some(), + "Junction should reference valid expanded_license_id" + ); + } + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test MD5-based deduplication in expanded_license +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_expanded_license_md5_deduplication( + ctx: &TrustifyContext, +) -> Result<(), anyhow::Error> { + // RED: Ingest once + ctx.ingest_document("spdx/OCP-TOOLS-4.11-RHEL-8.json") + .await?; + let count_first = expanded_license::Entity::find().count(&ctx.db).await?; + + // GREEN: Re-ingest same SBOM + ctx.ingest_document("spdx/OCP-TOOLS-4.11-RHEL-8.json") + .await?; + + // REFACTOR: Should have same count (MD5 index prevents duplicates) + let count_second = expanded_license::Entity::find().count(&ctx.db).await?; + assert_eq!( + count_first, count_second, + "MD5 hash should prevent duplicate expanded_text" + ); + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test that raw license.text is used for CycloneDX (no expansion) +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_cyclonedx_uses_raw_license_text(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let service = LicenseService::new(); + + // RED: Ingest CycloneDX + ctx.ingest_document("zookeeper-3.9.2-cyclonedx.json") + .await?; + + // GREEN: Verify CycloneDX DOES populate expanded_license (with raw text, not expanded) + // CycloneDX licenses don't have LicenseRef patterns, so expanded_text = raw license.text + let expanded_count = expanded_license::Entity::find().count(&ctx.db).await?; + assert!( + expanded_count > 0, + "CycloneDX should populate expanded_license with raw license text" + ); + + // Verify sbom_license_expanded junction is populated + let junction_count = sbom_license_expanded::Entity::find().count(&ctx.db).await?; + assert!( + junction_count > 0, + "CycloneDX should populate sbom_license_expanded junction table" + ); + + // REFACTOR: licenses() should return CycloneDX licenses via expanded_license path + let result = service + .licenses( + Query::default(), + Paginated { + offset: 0, + limit: 100, + }, + &ctx.db, + ) + .await?; + + let has_cyclonedx_license = result.items.iter().any(|l| { + l.license.contains("CDDL") || l.license.contains("GPL-2.0-with-classpath-exception") + }); + assert!( + has_cyclonedx_license, + "Should have CycloneDX licenses from license.text" + ); + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test refactored Func::coalesce() correctness +/// Verifies: Type-safe COALESCE(expanded_license.expanded_text, license.text) works correctly +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_coalesce_refactoring_correctness(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let service = LicenseService::new(); + + // RED: Before ingestion + // GREEN: Ingest SPDX (uses expanded_license via COALESCE) + let spdx_result = ctx + .ingest_document("spdx/OCP-TOOLS-4.11-RHEL-8.json") + .await?; + let sbom_id = spdx_result + .id + .try_as_uid() + .ok_or_else(|| anyhow::anyhow!("Expected UUID ID"))?; + + let info = service + .get_all_license_info(Id::Uuid(sbom_id), &ctx.db) + .await? + .expect("Should have license info"); + + // REFACTOR: Verify all licenses properly coalesced (no nulls, all expanded) + for mapping in &info { + assert!( + !mapping.license_name.is_empty(), + "COALESCE should never produce empty string" + ); + assert!( + !mapping.license_id.is_empty(), + "COALESCE should never produce empty string" + ); + + // Both columns use same COALESCE expression - should match + assert_eq!( + mapping.license_name, mapping.license_id, + "Both fields should use same COALESCE expression" + ); + } + + assert!(!info.is_empty(), "Should have at least one license"); + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test refactored Func::count() accuracy +/// Verifies: COUNT(*) replaced with Func::count(Expr::col(Asterisk)) produces correct counts +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_license_count_accuracy(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let service = LicenseService::new(); + + // RED: Baseline count before ingestion + let before = service + .licenses( + Query::default(), + Paginated { + offset: 0, + limit: 1000, + }, + &ctx.db, + ) + .await?; + let baseline_count = before.total; + + // GREEN: Ingest documents with licenses + ctx.ingest_documents([ + "spdx/OCP-TOOLS-4.11-RHEL-8.json", + "zookeeper-3.9.2-cyclonedx.json", + ]) + .await?; + + let after = service + .licenses( + Query::default(), + Paginated { + offset: 0, + limit: 1000, + }, + &ctx.db, + ) + .await?; + + // REFACTOR: Verify count increased and matches actual items + assert!( + after.total > baseline_count, + "Should have more licenses after ingestion" + ); + + let all_items = service + .licenses( + Query::default(), + Paginated { + offset: 0, + limit: 10000, + }, + &ctx.db, + ) + .await?; + assert_eq!( + all_items.total, + all_items.items.len() as u64, + "Total count should match number of items when all fetched" + ); + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test ORDER BY refactored COALESCE expression +/// Verifies: ORDER BY Func::coalesce() works correctly +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_license_ordering_by_coalesce(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let service = LicenseService::new(); + + // RED: No licenses before ingestion + // GREEN: Ingest multiple documents to get variety of licenses + ctx.ingest_documents([ + "spdx/OCP-TOOLS-4.11-RHEL-8.json", + "zookeeper-3.9.2-cyclonedx.json", + ]) + .await?; + + let result = service + .licenses( + Query::default(), + Paginated { + offset: 0, + limit: 100, + }, + &ctx.db, + ) + .await?; + + // REFACTOR: Verify licenses are ordered alphabetically via COALESCE + let licenses: Vec = result.items.iter().map(|l| l.license.clone()).collect(); + let mut sorted_licenses = licenses.clone(); + sorted_licenses.sort(); + + assert_eq!( + licenses, sorted_licenses, + "Licenses should be ordered alphabetically via COALESCE" + ); + assert!( + !licenses.is_empty(), + "Should have licenses to verify ordering" + ); + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test filtering on COALESCE result +/// Verifies: Filtering works on both expanded_text and raw text via COALESCE +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_license_filtering_on_coalesce(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + use trustify_common::db::query::q; + let service = LicenseService::new(); + + // RED: No licenses before ingestion + // GREEN: Ingest SPDX (expanded) and CycloneDX (raw) licenses + ctx.ingest_documents([ + "spdx/OCP-TOOLS-4.11-RHEL-8.json", + "zookeeper-3.9.2-cyclonedx.json", + ]) + .await?; + + // Search for Apache licenses (should find in both SPDX expanded and CycloneDX raw) + let apache_results = service + .licenses( + q("Apache"), + Paginated { + offset: 0, + limit: 100, + }, + &ctx.db, + ) + .await?; + + // REFACTOR: Verify filtering works on COALESCE result + assert!(apache_results.total > 0, "Should find Apache licenses"); + + for license in &apache_results.items { + assert!( + license.license.to_lowercase().contains("apache"), + "Filtered result should contain 'apache': {}", + license.license + ); + } + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test COALESCE NULL handling +/// Verifies: COALESCE properly handles NULLs (expanded_text NULL → fallback to license.text) +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_coalesce_null_handling(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let service = LicenseService::new(); + + // RED: No licenses before ingestion + // GREEN: Ingest CycloneDX (doesn't use expanded_license, so expanded_text is NULL) + ctx.ingest_document("zookeeper-3.9.2-cyclonedx.json") + .await?; + + let result = service + .licenses( + Query::default(), + Paginated { + offset: 0, + limit: 100, + }, + &ctx.db, + ) + .await?; + + // REFACTOR: Verify COALESCE fallback to license.text works when expanded_text is NULL + assert!( + result.total > 0, + "Should get CycloneDX licenses via COALESCE fallback to license.text" + ); + + for license in &result.items { + assert!( + !license.license.is_empty(), + "COALESCE should provide non-empty license text" + ); + } + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test pagination with UNION and refactored COUNT +/// Verifies: Paginated queries return correct subset and total count via Func::count() +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_license_pagination_with_count(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let service = LicenseService::new(); + + // RED: No licenses before ingestion + // GREEN: Ingest documents to create paginated results + ctx.ingest_documents([ + "spdx/OCP-TOOLS-4.11-RHEL-8.json", + "zookeeper-3.9.2-cyclonedx.json", + ]) + .await?; + + let page1 = service + .licenses( + Query::default(), + Paginated { + offset: 0, + limit: 5, + }, + &ctx.db, + ) + .await?; + let total = page1.total; + + // REFACTOR: Verify pagination works correctly with refactored COUNT + assert!(page1.items.len() <= 5, "Page 1 should have at most 5 items"); + assert!( + total >= page1.items.len() as u64, + "Total should be >= page 1 items" + ); + + let page2 = service + .licenses( + Query::default(), + Paginated { + offset: 5, + limit: 5, + }, + &ctx.db, + ) + .await?; + assert_eq!( + page2.total, total, + "Total count should be same across pages" + ); + + // Verify pages don't overlap + let page1_licenses: Vec = page1.items.iter().map(|l| l.license.clone()).collect(); + let page2_licenses: Vec = page2.items.iter().map(|l| l.license.clone()).collect(); + + for license in &page2_licenses { + assert!( + !page1_licenses.contains(license), + "Pages should not contain duplicate licenses" + ); + } + + Ok(()) +} + +/// Test that pre-loaded SPDX dictionary entries appear in license listing +/// Verifies: LEFT JOIN on sbom_package_license allows pre-loaded licenses to be visible +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_preloaded_licenses_visible(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let service = LicenseService::new(); + + // Baseline: Get initial license count (includes pre-loaded SPDX dictionary) + let before = service + .licenses(Query::default(), Paginated::default(), &ctx.db) + .await?; + + // Pre-loaded licenses should be visible even before any SBOM ingestion + assert!( + before.total > 0, + "Pre-loaded SPDX dictionary entries should be visible" + ); + + Ok(()) +} diff --git a/modules/fundamental/src/purl/model/details/purl.rs b/modules/fundamental/src/purl/model/details/purl.rs index 6ac0bf8ec..00813d640 100644 --- a/modules/fundamental/src/purl/model/details/purl.rs +++ b/modules/fundamental/src/purl/model/details/purl.rs @@ -1,7 +1,7 @@ use crate::{ Error, advisory::model::AdvisoryHead, - common::{LicenseInfo, LicenseRefMapping, license_filtering}, + common::{LicenseInfo, LicenseRefMapping, license_filtering::license_text_coalesce}, purl::model::{BasePurlHead, PurlHead, VersionedPurlHead}, sbom::{model::SbomHead, service::sbom::LicenseBasicInfo}, vulnerability::model::VulnerabilityHead, @@ -23,9 +23,9 @@ use trustify_common::{ use trustify_cvss::cvss3::{Cvss3Base, score::Score, severity::Severity}; use trustify_entity::{ advisory, base_purl, cpe, cvss3, license, organization, product, product_status, - product_version, product_version_range, purl_status, qualified_purl, sbom, sbom_package, - sbom_package_license, sbom_package_purl_ref, status, version_range, versioned_purl, - vulnerability, + product_version, product_version_range, purl_status, qualified_purl, sbom, + sbom_license_expanded, sbom_package, sbom_package_license, sbom_package_purl_ref, status, + version_range, versioned_purl, vulnerability, }; use trustify_module_ingestor::common::{Deprecation, DeprecationForExt}; use utoipa::ToSchema; @@ -109,10 +109,7 @@ impl PurlDetails { let licenses: Vec = sbom_package_purl_ref::Entity::find() .distinct() .select_only() - .column_as( - license_filtering::get_case_license_text_sbom_id(), - "license_name", - ) + .column_as(license_text_coalesce(), "license_name") .select_column(sbom_package_license::Column::LicenseType) .filter(sbom_package_purl_ref::Column::QualifiedPurlId.eq(qualified_package.id)) .join( @@ -121,7 +118,15 @@ impl PurlDetails { ) .join(JoinType::Join, sbom_package::Relation::PackageLicense.def()) .join( - JoinType::Join, + JoinType::LeftJoin, + sbom_package_license::Relation::SbomLicenseExpanded.def(), + ) + .join( + JoinType::LeftJoin, + sbom_license_expanded::Relation::ExpandedLicense.def(), + ) + .join( + JoinType::LeftJoin, sbom_package_license::Relation::License.def(), ) .into_model::() diff --git a/modules/fundamental/src/purl/service/mod.rs b/modules/fundamental/src/purl/service/mod.rs index 7750aad2d..4a2321679 100644 --- a/modules/fundamental/src/purl/service/mod.rs +++ b/modules/fundamental/src/purl/service/mod.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use crate::{ Error, - common::license_filtering::{LICENSE, build_license_filtering_with_clause}, + common::license_filtering::LICENSE, purl::model::{ RecommendEntry, VulnerabilityStatus, details::{ @@ -13,12 +13,10 @@ use crate::{ }; use regex::Regex; use sea_orm::{ - ColumnTrait, ConnectionTrait, DbBackend, EntityTrait, FromQueryResult, QueryFilter, QueryOrder, - QuerySelect, QueryTrait, RelationTrait, Statement, prelude::Uuid, -}; -use sea_query::{ - Alias, ColumnType, Condition, Expr, JoinType, Order, PgFunc, PostgresQueryBuilder, + ColumnTrait, ConnectionTrait, EntityTrait, FromQueryResult, QueryFilter, QueryOrder, + QuerySelect, QueryTrait, RelationTrait, prelude::Uuid, }; +use sea_query::{ColumnType, JoinType, Order}; use tracing::instrument; use trustify_common::{ db::{ @@ -31,7 +29,8 @@ use trustify_common::{ use trustify_entity::{ base_purl, license, qualified_purl::{self, CanonicalPurl}, - sbom_package, sbom_package_license, sbom_package_purl_ref, versioned_purl, + sbom_license_expanded, sbom_package, sbom_package_license, sbom_package_purl_ref, + versioned_purl, }; use trustify_module_ingestor::common::Deprecation; @@ -322,23 +321,27 @@ impl PurlService { .get_constraint_for_field(LICENSE) .map(|constraint| q(&format!("{constraint}"))) { - #[derive(Debug, FromQueryResult)] - struct QualifiedPurlIdResult { - id: Uuid, - } - - // Build the CTEs for license filtering - let with_clause = build_license_filtering_with_clause(); - - let mut statement = sbom_package_purl_ref::Entity::find() - .distinct() + // SPDX path: join through junction → dictionary + let spdx_subquery = sbom_package_purl_ref::Entity::find() .select_only() - .column_as(sbom_package_purl_ref::Column::QualifiedPurlId, "id") + .distinct() + .column(sbom_package_purl_ref::Column::QualifiedPurlId) .join( - JoinType::Join, + JoinType::InnerJoin, sbom_package_purl_ref::Relation::Package.def(), ) - .join(JoinType::Join, sbom_package::Relation::PackageLicense.def()) + .join( + JoinType::InnerJoin, + sbom_package::Relation::PackageLicense.def(), + ) + .join( + JoinType::InnerJoin, + sbom_package_license::Relation::SbomLicenseExpanded.def(), + ) + .join( + JoinType::InnerJoin, + sbom_license_expanded::Relation::ExpandedLicense.def(), + ) .filtering_with( license_query.clone(), Columns::default() @@ -349,52 +352,22 @@ impl PurlService { }), )? .into_query(); - let x = statement - .join( - JoinType::Join, - Alias::new("expanded"), - Condition::all() - .add( - Expr::col(( - sbom_package_license::Entity, - sbom_package_license::Column::SbomId, - )) - .equals((Alias::new("expanded"), Alias::new("sbom_id"))), - ) - .add( - Expr::col(( - sbom_package_license::Entity, - sbom_package_license::Column::LicenseId, - )) - .equals((Alias::new("expanded"), Alias::new("license_id"))), - ), - ) - .to_owned(); - let main_query = x.with(with_clause); - let (sql, values) = main_query.build(PostgresQueryBuilder); - let qualified_purl_ids_filtered_by_license: Vec = - QualifiedPurlIdResult::find_by_statement(Statement::from_sql_and_values( - DbBackend::Postgres, - sql, - values, - )) - .all(connection) - .await? - .into_iter() - .map(|r| r.id) - .collect(); + // CycloneDX path: direct text match let cyclonedx_subquery = sbom_package_purl_ref::Entity::find() - .distinct() .select_only() + .distinct() .column(sbom_package_purl_ref::Column::QualifiedPurlId) .join( - JoinType::Join, + JoinType::InnerJoin, sbom_package_purl_ref::Relation::Package.def(), ) - .join(JoinType::Join, sbom_package::Relation::PackageLicense.def()) .join( - JoinType::Join, + JoinType::InnerJoin, + sbom_package::Relation::PackageLicense.def(), + ) + .join( + JoinType::InnerJoin, sbom_package_license::Relation::License.def(), ) .filtering_with( @@ -408,14 +381,12 @@ impl PurlService { )? .into_query(); - // Combine SPDX and CycloneDX results - let combined_condition = Condition::any() - .add( - Expr::col((qualified_purl::Entity, qualified_purl::Column::Id)) - .eq(PgFunc::any(qualified_purl_ids_filtered_by_license)), - ) - .add(qualified_purl::Column::Id.in_subquery(cyclonedx_subquery)); - select = select.filter(combined_condition); + // Apply as subquery filter + select = select.filter( + qualified_purl::Column::Id + .in_subquery(spdx_subquery) + .or(qualified_purl::Column::Id.in_subquery(cyclonedx_subquery)), + ); } let limiter = select.limiting(connection, paginated.offset, paginated.limit); diff --git a/modules/fundamental/src/sbom/model/details.rs b/modules/fundamental/src/sbom/model/details.rs index 3b243531f..3028db8da 100644 --- a/modules/fundamental/src/sbom/model/details.rs +++ b/modules/fundamental/src/sbom/model/details.rs @@ -441,6 +441,7 @@ pub struct SbomAdvisory { } impl SbomAdvisory { + #[allow(deprecated)] #[instrument(skip_all, err(level=tracing::Level::INFO))] pub async fn from_models( statuses: Vec, diff --git a/modules/fundamental/src/sbom/model/mod.rs b/modules/fundamental/src/sbom/model/mod.rs index 10f220bfe..09d301814 100644 --- a/modules/fundamental/src/sbom/model/mod.rs +++ b/modules/fundamental/src/sbom/model/mod.rs @@ -132,6 +132,11 @@ pub struct SbomPackage { #[cfg_attr(feature = "async-graphql", graphql(skip))] pub licenses: Vec, /// LicenseRef mappings + /// + /// **Deprecated**: Licenses are now pre-expanded at ingestion time via `expanded_license` / + /// `sbom_license_expanded` tables. This field is always empty and will be removed in a future + /// release. + #[deprecated(note = "Licenses are pre-expanded; this field is always empty")] #[cfg_attr(feature = "async-graphql", graphql(skip))] pub licenses_ref_mapping: Vec, } diff --git a/modules/fundamental/src/sbom/service/sbom.rs b/modules/fundamental/src/sbom/service/sbom.rs index 3e302e33f..b9c313378 100644 --- a/modules/fundamental/src/sbom/service/sbom.rs +++ b/modules/fundamental/src/sbom/service/sbom.rs @@ -1,13 +1,7 @@ use super::SbomService; use crate::{ Error, - common::{ - LicenseRefMapping, - license_filtering::{ - LICENSE, apply_license_filtering, build_license_filtering_with_clause, - create_sbom_package_license_filtering_base_query, get_case_license_text_sbom_id, - }, - }, + common::license_filtering::{LICENSE, license_text_coalesce}, sbom::model::{ SbomExternalPackageReference, SbomNodeReference, SbomPackage, SbomPackageRelation, SbomSummary, Which, details::SbomDetails, @@ -15,21 +9,14 @@ use crate::{ }; use futures_util::{StreamExt, TryStreamExt, stream}; use sea_orm::{ - ColumnTrait, ConnectionTrait, DbBackend, DbErr, EntityTrait, FromJsonQueryResult, - FromQueryResult, IntoSimpleExpr, QueryFilter, QueryOrder, QueryResult, QuerySelect, QueryTrait, - RelationTrait, Select, SelectColumns, Statement, StreamTrait, prelude::Uuid, -}; -use sea_query::{ - Alias, ColumnType, Condition, Expr, JoinType, PgFunc, PostgresQueryBuilder, - extension::postgres::PgExpr, + ColumnTrait, ConnectionTrait, DbErr, EntityTrait, FromJsonQueryResult, FromQueryResult, + IntoSimpleExpr, QueryFilter, QueryOrder, QueryResult, QuerySelect, QueryTrait, RelationTrait, + Select, SelectColumns, Statement, StreamTrait, prelude::Uuid, }; +use sea_query::{ColumnType, Expr, JoinType, extension::postgres::PgExpr}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::{ - collections::{BTreeMap, HashMap}, - fmt::Debug, - sync::Arc, -}; +use std::{collections::HashMap, fmt::Debug, sync::Arc}; use tracing::instrument; use trustify_common::{ cpe::Cpe, @@ -47,12 +34,12 @@ use trustify_entity::{ advisory, advisory_vulnerability, base_purl, cpe::{self, CpeDto}, labels::Labels, - license, licensing_infos, organization, package_relates_to_package, + license, organization, package_relates_to_package, qualified_purl::{self, CanonicalPurl}, relationship::Relationship, sbom::{self, SbomNodeLink}, - sbom_node, sbom_package, sbom_package_cpe_ref, sbom_package_license, sbom_package_purl_ref, - source_document, status, versioned_purl, vulnerability, + sbom_license_expanded, sbom_node, sbom_package, sbom_package_cpe_ref, sbom_package_license, + sbom_package_purl_ref, source_document, status, versioned_purl, vulnerability, }; impl SbomService { @@ -199,21 +186,14 @@ impl SbomService { .get_constraint_for_field(LICENSE) .map(|constraint| q(&format!("{constraint}"))) { - #[derive(Debug, FromQueryResult)] - struct QualifiedPurlIdResult { - id: Uuid, - } - - // Build the CTEs for license filtering - let with_clause = build_license_filtering_with_clause(); - - let mut statement = sbom_package_license::Entity::find() - .distinct() + // SPDX path: join through junction → dictionary + let spdx_subquery = sbom_license_expanded::Entity::find() .select_only() - .column_as(sbom_package_license::Column::SbomId, "id") + .distinct() + .column(sbom_license_expanded::Column::SbomId) .join( - JoinType::Join, - sbom_package_license::Relation::License.def(), + JoinType::InnerJoin, + sbom_license_expanded::Relation::ExpandedLicense.def(), ) .filtering_with( license_query.clone(), @@ -225,46 +205,14 @@ impl SbomService { }), )? .into_query(); - let x = statement - .join( - JoinType::Join, - Alias::new("expanded"), - Condition::all() - .add( - Expr::col(( - sbom_package_license::Entity, - sbom_package_license::Column::SbomId, - )) - .equals((Alias::new("expanded"), Alias::new("sbom_id"))), - ) - .add( - Expr::col(( - sbom_package_license::Entity, - sbom_package_license::Column::LicenseId, - )) - .equals((Alias::new("expanded"), Alias::new("license_id"))), - ), - ) - .to_owned(); - let main_query = x.with(with_clause); - let (sql, values) = main_query.build(PostgresQueryBuilder); - let qualified_purl_ids_filtered_by_license: Vec = - QualifiedPurlIdResult::find_by_statement(Statement::from_sql_and_values( - DbBackend::Postgres, - sql, - values, - )) - .all(connection) - .await? - .into_iter() - .map(|r| r.id) - .collect(); + // CycloneDX path: direct text match let cyclonedx_subquery = sbom_package_license::Entity::find() .select_only() + .distinct() .column(sbom_package_license::Column::SbomId) .join( - JoinType::Join, + JoinType::InnerJoin, sbom_package_license::Relation::License.def(), ) .filtering_with( @@ -278,14 +226,12 @@ impl SbomService { )? .into_query(); - // Combine SPDX and CycloneDX results - let combined_condition = Condition::any() - .add( - Expr::col((sbom::Entity, sbom::Column::SbomId)) - .eq(PgFunc::any(qualified_purl_ids_filtered_by_license)), - ) - .add(sbom::Column::SbomId.in_subquery(cyclonedx_subquery)); - query = query.filter(combined_condition); + // Apply as subquery filter + query = query.filter( + sbom::Column::SbomId + .in_subquery(spdx_subquery) + .or(sbom::Column::SbomId.in_subquery(cyclonedx_subquery)), + ); } let limiter = query @@ -348,13 +294,66 @@ impl SbomService { query = join_licenses(query); - // Add license filtering if license query is present - query = apply_license_filtering( - query, - &search, - || create_sbom_package_license_filtering_base_query(sbom_id), - sbom_package::Column::NodeId, - )?; + // Apply license filter via subqueries, matching the same pattern as `fetch_sboms`. + // The `filtering_with` translator cannot express OR across two different table columns, + // so we pre-filter node_ids: any package whose SPDX-expanded text OR raw license text + // matches the query is included. + if let Some(license_constraint) = search + .get_constraint_for_field(LICENSE) + .map(|constraint| q(&format!("{constraint}"))) + { + // SPDX path: match via expanded_license dictionary + let spdx_pkg_subquery = sbom_package_license::Entity::find() + .select_only() + .distinct() + .column(sbom_package_license::Column::NodeId) + .join( + JoinType::InnerJoin, + sbom_package_license::Relation::SbomLicenseExpanded.def(), + ) + .join( + JoinType::InnerJoin, + sbom_license_expanded::Relation::ExpandedLicense.def(), + ) + .filter(sbom_package_license::Column::SbomId.eq(sbom_id)) + .filtering_with( + license_constraint.clone(), + Columns::default() + .add_column("expanded_text", ColumnType::Text) + .translator(|field, operator, value| match field { + LICENSE => Some(format!("expanded_text{operator}{value}")), + _ => None, + }), + )? + .into_query(); + + // CycloneDX path: match raw license text directly + let cdx_pkg_subquery = sbom_package_license::Entity::find() + .select_only() + .distinct() + .column(sbom_package_license::Column::NodeId) + .join( + JoinType::InnerJoin, + sbom_package_license::Relation::License.def(), + ) + .filter(sbom_package_license::Column::SbomId.eq(sbom_id)) + .filtering_with( + license_constraint, + license::Entity + .columns() + .translator(|field, operator, value| match field { + LICENSE => Some(format!("text{operator}{value}")), + _ => None, + }), + )? + .into_query(); + + query = query.filter( + sbom_package::Column::NodeId + .in_subquery(spdx_pkg_subquery) + .or(sbom_package::Column::NodeId.in_subquery(cdx_pkg_subquery)), + ); + } query = join_purls_and_cpes(query) .filtering_with( @@ -369,8 +368,8 @@ impl SbomService { .add_columns(sbom_package_purl_ref::Entity) .translator(|field, _operator, _value| { match field { - // Add an empty condition (effectively TRUE) to the main SQL query - // since the real filtering by license happens in the license subqueries above + // License filtering is handled via subqueries above; return an empty + // condition here so the main query is not further restricted. LICENSE => Some("".to_string()), _ => None, } @@ -394,29 +393,12 @@ impl SbomService { .fetch() .await? .into_iter() - .map(|row| package_from_row(row, BTreeMap::new())) + .map(package_from_row) .collect(); Ok(PaginatedResults { items, total }) } - /// Get all the tuples License ID, License Name from the licensing_infos table for a single SBOM - #[instrument(skip(connection), err(level=tracing::Level::INFO))] - pub async fn get_licensing_infos( - connection: &C, - sbom_id: Uuid, - ) -> Result, Error> { - let licensing_infos_result: Vec<(String, String)> = licensing_infos::Entity::find() - .select_only() - .column(licensing_infos::Column::LicenseId) - .column(licensing_infos::Column::Name) - .filter(licensing_infos::Column::SbomId.eq(sbom_id)) - .into_tuple() - .all(connection) - .await?; - Ok(licensing_infos_result.into_iter().collect()) - } - /// Get all packages describing the SBOM. #[instrument(skip(self, db), err(level=tracing::Level::INFO))] pub async fn describes_packages( @@ -656,14 +638,12 @@ impl SbomService { // execute - let licensing_infos = Self::get_licensing_infos(db, sbom_id).await?; - let r: R::Output = R::get(options, db, query).await?; Ok(r.flat_map_all(|row| { Some(SbomPackageRelation { relationship: row.relationship?, - package: package_from_row(row, licensing_infos.clone()), + package: package_from_row(row), }) })) } @@ -769,9 +749,9 @@ where Expr::cust_with_exprs( "coalesce(json_agg(distinct jsonb_build_object('license_name', $1, 'license_type', $2)) filter (where $3), '[]'::json)", [ - get_case_license_text_sbom_id(), + license_text_coalesce(), sbom_package_license::Column::LicenseType.into_simple_expr(), - license::Column::Text.is_not_null().into_simple_expr(), + Expr::col((license::Entity, license::Column::Text)).is_not_null(), ], ), "licenses", @@ -779,10 +759,19 @@ where .join( JoinType::LeftJoin, sbom_package::Relation::PackageLicense.def(), - ).join( - JoinType::LeftJoin, - sbom_package_license::Relation::License.def(), - ) + ) + .join( + JoinType::LeftJoin, + sbom_package_license::Relation::SbomLicenseExpanded.def(), + ) + .join( + JoinType::LeftJoin, + sbom_license_expanded::Relation::ExpandedLicense.def(), + ) + .join( + JoinType::LeftJoin, + sbom_package_license::Relation::License.def(), + ) } #[derive(FromQueryResult)] @@ -806,7 +795,8 @@ pub struct LicenseBasicInfo { } /// Convert values from a "package row" into an SBOM package -fn package_from_row(row: PackageCatcher, licensing_infos: BTreeMap) -> SbomPackage { +#[allow(deprecated)] +fn package_from_row(row: PackageCatcher) -> SbomPackage { let purl = row .purls .into_iter() @@ -843,31 +833,12 @@ fn package_from_row(row: PackageCatcher, licensing_infos: BTreeMap Result<(), anyhow::Error> { + let service = SbomService::new(ctx.db.clone()); + + // RED: No packages before ingestion + // GREEN: Ingest SPDX (uses expanded_license) and CycloneDX (uses raw license.text) + let spdx_result = ctx + .ingest_document("spdx/OCP-TOOLS-4.11-RHEL-8.json") + .await?; + let cyclonedx_result = ctx + .ingest_document("zookeeper-3.9.2-cyclonedx.json") + .await?; + + let spdx_id = spdx_result + .id + .try_as_uid() + .ok_or_else(|| anyhow::anyhow!("Expected UUID ID"))?; + let cyclonedx_id = cyclonedx_result + .id + .try_as_uid() + .ok_or_else(|| anyhow::anyhow!("Expected UUID ID"))?; + + let spdx_packages = service + .fetch_sbom_packages( + spdx_id, + Query::default(), + Paginated { + offset: 0, + limit: 100, + }, + &ctx.db, + ) + .await?; + + let cyclonedx_packages = service + .fetch_sbom_packages( + cyclonedx_id, + Query::default(), + Paginated { + offset: 0, + limit: 100, + }, + &ctx.db, + ) + .await?; + + // REFACTOR: Verify SPDX licenses expanded via COALESCE (no LicenseRef-) + assert!( + !spdx_packages.items.is_empty(), + "SPDX SBOM should have packages" + ); + + for package in &spdx_packages.items { + for license in &package.licenses { + assert!( + !license.license_name.contains("LicenseRef-"), + "SPDX licenses should be expanded via COALESCE: {}", + license.license_name + ); + } + } + + // Verify CycloneDX licenses exist via COALESCE fallback to license.text + assert!( + !cyclonedx_packages.items.is_empty(), + "CycloneDX SBOM should have packages" + ); + + let has_licenses = cyclonedx_packages + .items + .iter() + .any(|p| !p.licenses.is_empty()); + assert!( + has_licenses, + "CycloneDX packages should have licenses via COALESCE fallback" + ); + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test package license filtering with COALESCE +/// Verifies: License filtering works on both expanded and raw licenses via COALESCE +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_sbom_package_license_filtering_with_coalesce( + ctx: &TrustifyContext, +) -> Result<(), anyhow::Error> { + let service = SbomService::new(ctx.db.clone()); + + // RED: No packages before ingestion + // GREEN: Ingest SPDX with Apache licenses + let result = ctx + .ingest_document("spdx/OCP-TOOLS-4.11-RHEL-8.json") + .await?; + let sbom_id = result + .id + .try_as_uid() + .ok_or_else(|| anyhow::anyhow!("Expected UUID ID"))?; + + // Filter by license (should work via COALESCE on expanded_text OR text) + let apache_packages = service + .fetch_sbom_packages( + sbom_id, + q("license~Apache"), + Paginated { + offset: 0, + limit: 100, + }, + &ctx.db, + ) + .await?; + + // REFACTOR: Verify filtering works on COALESCE result + assert!( + apache_packages.total > 0, + "Expected at least one package to match Apache filter" + ); + let has_apache = apache_packages.items.iter().any(|p| { + p.licenses + .iter() + .any(|l| l.license_name.to_lowercase().contains("apache")) + }); + assert!( + has_apache, + "Filtered packages should contain Apache licenses" + ); + + Ok(()) +} + +/// RED-GREEN-REFACTOR: Test refactored IS NOT NULL filter +/// Verifies: Expr::col().is_not_null() works correctly in license JSON aggregation +#[test_context(TrustifyContext)] +#[test(tokio::test)] +async fn test_sbom_package_license_not_null_filter( + ctx: &TrustifyContext, +) -> Result<(), anyhow::Error> { + let service = SbomService::new(ctx.db.clone()); + + // RED: No packages before ingestion + // GREEN: Ingest SBOM with licenses + let result = ctx + .ingest_document("spdx/OCP-TOOLS-4.11-RHEL-8.json") + .await?; + let sbom_id = result + .id + .try_as_uid() + .ok_or_else(|| anyhow::anyhow!("Expected UUID ID"))?; + + let packages = service + .fetch_sbom_packages( + sbom_id, + Query::default(), + Paginated { + offset: 0, + limit: 1000, + }, + &ctx.db, + ) + .await?; + + // REFACTOR: Verify IS NOT NULL filter works (refactored to Expr::col().is_not_null()) + for package in &packages.items { + // If a package has licenses, none should be empty (due to IS NOT NULL filter) + for license in &package.licenses { + assert!( + !license.license_name.is_empty(), + "License should not be empty due to IS NOT NULL filter in join_licenses()" + ); + } + } + + Ok(()) +} diff --git a/modules/fundamental/tests/sbom/spdx/perf.rs b/modules/fundamental/tests/sbom/spdx/perf.rs index df9ff0735..a1ae60505 100644 --- a/modules/fundamental/tests/sbom/spdx/perf.rs +++ b/modules/fundamental/tests/sbom/spdx/perf.rs @@ -1,3 +1,5 @@ +#![allow(deprecated)] + use super::*; use test_context::test_context; use test_log::test; diff --git a/modules/importer/src/lib.rs b/modules/importer/src/lib.rs index 51c2538b0..88eee266b 100644 --- a/modules/importer/src/lib.rs +++ b/modules/importer/src/lib.rs @@ -1,4 +1,4 @@ -#![recursion_limit = "256"] +#![recursion_limit = "512"] pub mod endpoints; pub mod model; diff --git a/modules/ingestor/src/graph/sbom/common/expanded_license.rs b/modules/ingestor/src/graph/sbom/common/expanded_license.rs new file mode 100644 index 000000000..cfce4c3b6 --- /dev/null +++ b/modules/ingestor/src/graph/sbom/common/expanded_license.rs @@ -0,0 +1,88 @@ +use sea_orm::{ConnectionTrait, DbErr, Statement}; +use uuid::Uuid; + +/// Populates expanded_license and sbom_license_expanded tables during SBOM ingestion +/// +/// This function uses two SQL statements to: +/// 1. Call expand_license_expression_with_mappings() once per license +/// 2. Insert distinct expanded texts into the expanded_license dictionary +/// 3. Populate the sbom_license_expanded junction table +/// +/// Raw SQL is used because the query involves: +/// - PostgreSQL composite type `license_mapping` constructed with `ROW(...)` +/// - Array aggregation `array_agg()` over composite types +/// - Custom PL/pgSQL function `expand_license_expression_with_mappings()` +/// - Complex CTEs with multiple insert operations +/// +/// While SeaORM could express this via custom expressions, it would be significantly +/// more verbose and harder to maintain than the raw SQL. +/// +/// **Transaction safety**: This function expects to be called within a transaction +/// (as it always is during SBOM ingestion). Both SQL statements will be atomic +/// within the caller's transaction. +/// +/// **Note on SQL duplication**: Similar SQL appears in migration m0002120 for backfilling +/// existing data. The migration processes ALL SBOMs at once, while this function runs +/// per-SBOM during ingestion. Keep both in sync when updating license expansion logic. +pub async fn populate_expanded_license( + sbom_id: Uuid, + db: &impl ConnectionTrait, +) -> Result<(), DbErr> { + // Step 1: Insert into expanded_license dictionary + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + r#" +INSERT INTO expanded_license (expanded_text) +SELECT DISTINCT expand_license_expression_with_mappings( + l.text, + COALESCE(lim.license_mapping, ARRAY[]::license_mapping[]) +) +FROM sbom_package_license spl +JOIN license l ON l.id = spl.license_id +LEFT JOIN ( + SELECT array_agg(ROW(license_id, name)::license_mapping) AS license_mapping, sbom_id + FROM licensing_infos + GROUP BY sbom_id +) lim ON lim.sbom_id = spl.sbom_id +WHERE spl.sbom_id = $1 +ON CONFLICT (text_hash) DO NOTHING + "#, + [sbom_id.into()], + )) + .await?; + + // Step 2: Insert into sbom_license_expanded junction table + // Use CTE to call expand_license_expression_with_mappings() only once per (sbom_id, license_id) + db.execute(Statement::from_sql_and_values( + db.get_database_backend(), + r#" +WITH license_expansions AS ( + SELECT DISTINCT + spl.sbom_id, + spl.license_id, + expand_license_expression_with_mappings( + l.text, + COALESCE(lim.license_mapping, ARRAY[]::license_mapping[]) + ) AS expanded_text + FROM sbom_package_license spl + JOIN license l ON l.id = spl.license_id + LEFT JOIN ( + SELECT array_agg(ROW(license_id, name)::license_mapping) AS license_mapping, sbom_id + FROM licensing_infos + GROUP BY sbom_id + ) lim ON lim.sbom_id = spl.sbom_id + WHERE spl.sbom_id = $1 +) +INSERT INTO sbom_license_expanded (sbom_id, license_id, expanded_license_id) +SELECT le.sbom_id, le.license_id, el.id +FROM license_expansions le +JOIN expanded_license el ON el.text_hash = md5(le.expanded_text) +ON CONFLICT (sbom_id, license_id) DO UPDATE +SET expanded_license_id = EXCLUDED.expanded_license_id + "#, + [sbom_id.into()], + )) + .await?; + + Ok(()) +} diff --git a/modules/ingestor/src/graph/sbom/common/mod.rs b/modules/ingestor/src/graph/sbom/common/mod.rs index 8ed48e4ea..4da4761a6 100644 --- a/modules/ingestor/src/graph/sbom/common/mod.rs +++ b/modules/ingestor/src/graph/sbom/common/mod.rs @@ -1,4 +1,5 @@ mod checksum; +mod expanded_license; mod external; mod file; mod license; @@ -8,6 +9,7 @@ mod package; mod relationship; pub use checksum::*; +pub use expanded_license::*; pub use external::*; pub use file::*; pub use license::*; diff --git a/modules/ingestor/src/graph/sbom/cyclonedx.rs b/modules/ingestor/src/graph/sbom/cyclonedx.rs index 31f0e5954..22e3b3cd1 100644 --- a/modules/ingestor/src/graph/sbom/cyclonedx.rs +++ b/modules/ingestor/src/graph/sbom/cyclonedx.rs @@ -6,7 +6,7 @@ use crate::{ sbom::{ CycloneDx as CycloneDxProcessor, LicenseCreator, LicenseInfo, NodeInfoParam, PackageCreator, PackageLicensenInfo, PackageReference, References, RelationshipCreator, - SbomContext, SbomInformation, + SbomContext, SbomInformation, populate_expanded_license, processor::{ InitContext, PostContext, Processor, RedHatProductComponentRelationships, RunProcessors, @@ -134,11 +134,11 @@ impl<'a> From> for SbomInformation { impl SbomContext { #[instrument(skip(connection, sbom, warnings), err(level=tracing::Level::INFO))] - pub async fn ingest_cyclonedx( + pub async fn ingest_cyclonedx( &self, mut sbom: Box, warnings: &dyn ReportSink, - connection: &C, + connection: &impl ConnectionTrait, ) -> Result<(), Error> { // pre-flight checks @@ -340,6 +340,9 @@ impl<'a> Creator<'a> { packages.create(db).await?; relationships.create(db).await?; + // Populate expanded license tables + populate_expanded_license(self.sbom_id, db).await?; + // done Ok(()) diff --git a/modules/ingestor/src/graph/sbom/spdx.rs b/modules/ingestor/src/graph/sbom/spdx.rs index 106e84251..57bdf5911 100644 --- a/modules/ingestor/src/graph/sbom/spdx.rs +++ b/modules/ingestor/src/graph/sbom/spdx.rs @@ -6,7 +6,7 @@ use crate::{ sbom::{ FileCreator, LicenseCreator, LicenseInfo, LicensingInfo, LicensingInfoCreator, NodeInfoParam, PackageCreator, PackageLicensenInfo, PackageReference, References, - RelationshipCreator, SbomContext, SbomInformation, Spdx, + RelationshipCreator, SbomContext, SbomInformation, Spdx, populate_expanded_license, processor::{ InitContext, PostContext, Processor, RedHatProductComponentRelationships, RunProcessors, @@ -102,11 +102,11 @@ impl<'a> From> for SbomInformation { impl SbomContext { #[instrument(skip(db, sbom_data, warnings), ret(level=tracing::Level::DEBUG))] - pub async fn ingest_spdx( + pub async fn ingest_spdx( &self, sbom_data: SPDX, warnings: &dyn ReportSink, - db: &C, + db: &impl ConnectionTrait, ) -> Result<(), Error> { // pre-flight checks @@ -357,6 +357,9 @@ impl SbomContext { files.create(db).await?; relationships.create(db).await?; + // Populate expanded license tables + populate_expanded_license(self.sbom.sbom_id, db).await?; + // done Ok(()) diff --git a/modules/ingestor/tests/parallel.rs b/modules/ingestor/tests/parallel.rs index 172958355..80358e816 100644 --- a/modules/ingestor/tests/parallel.rs +++ b/modules/ingestor/tests/parallel.rs @@ -1,3 +1,4 @@ +#![recursion_limit = "512"] //! Testing parallel operations use bytes::Bytes; diff --git a/modules/user/src/lib.rs b/modules/user/src/lib.rs index ad8476b07..95beafcad 100644 --- a/modules/user/src/lib.rs +++ b/modules/user/src/lib.rs @@ -1,4 +1,4 @@ -#![recursion_limit = "256"] +#![recursion_limit = "512"] pub mod endpoints; pub mod service; diff --git a/openapi.yaml b/openapi.yaml index bb244183b..62cc44a19 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4347,7 +4347,13 @@ components: type: array items: $ref: '#/components/schemas/LicenseRefMapping' - description: LicenseRef mappings + description: |- + LicenseRef mappings + + **Deprecated**: Licenses are now pre-expanded at ingestion time via `expanded_license` / + `sbom_license_expanded` tables. This field is always empty and will be removed in a future + release. + deprecated: true name: type: string description: The name of the package in the SBOM @@ -4957,7 +4963,13 @@ components: type: array items: $ref: '#/components/schemas/LicenseRefMapping' - description: LicenseRef mappings + description: |- + LicenseRef mappings + + **Deprecated**: Licenses are now pre-expanded at ingestion time via `expanded_license` / + `sbom_license_expanded` tables. This field is always empty and will be removed in a future + release. + deprecated: true name: type: string description: The name of the package in the SBOM diff --git a/server/src/lib.rs b/server/src/lib.rs index 57398951b..9413b8ba3 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,4 +1,4 @@ -#![recursion_limit = "256"] +#![recursion_limit = "512"] #[cfg(feature = "garage-door")] mod embedded_oidc; diff --git a/trustd/src/main.rs b/trustd/src/main.rs index 4a83b5510..0f75563e2 100644 --- a/trustd/src/main.rs +++ b/trustd/src/main.rs @@ -1,4 +1,4 @@ -#![recursion_limit = "256"] +#![recursion_limit = "512"] use clap::Parser; use std::{ diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7302b77c3..9f55dc507 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,5 +1,5 @@ #![allow(clippy::unwrap_used)] -#![recursion_limit = "256"] +#![recursion_limit = "512"] use crate::log::init_log; use clap::{Parser, Subcommand};