diff --git a/Changelog.md b/Changelog.md index fe9f72312c..27da12a82f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,9 @@ # FOSSA CLI Changelog +## 3.17.14 + +- Maven: Recover dependency edges that Maven's resolution omits as duplicates, so transitive dependencies shared by multiple parents are attributed to all of them instead of a single winning parent. ([#1730](https://github.com/fossas/fossa-cli/pull/1730)) + ## 3.17.13 - Licensing: Detect the Lucky Penny Software RPL-1.5 dual-license notice (AutoMapper, MediatR), resolving it to `rpl-1.5 OR proprietary-license` instead of `unknown`. ([#1729](https://github.com/fossas/fossa-cli/pull/1729)) diff --git a/src/Strategy/Maven/Plugin.hs b/src/Strategy/Maven/Plugin.hs index 64e19a8216..34febd511e 100644 --- a/src/Strategy/Maven/Plugin.hs +++ b/src/Strategy/Maven/Plugin.hs @@ -5,6 +5,7 @@ module Strategy.Maven.Plugin ( withUnpackedPlugin, installPlugin, parsePluginOutput, + parseVerboseGraphs, depGraphPlugin, depGraphPluginLegacy, Artifact (..), @@ -13,10 +14,15 @@ module Strategy.Maven.Plugin ( PluginOutput (..), ReactorOutput (..), ReactorArtifact (..), + VerboseArtifact (..), + VerboseEdge (..), + VerboseGraph (..), + augmentWithDuplicateEdges, parseReactorOutput, textArtifactToPluginOutput, execPluginAggregate, execPluginReactor, + execPluginVerboseGraph, mavenCmdCandidates, ) where @@ -36,13 +42,13 @@ import Data.List.NonEmpty (NonEmpty (..)) import Data.List.NonEmpty qualified as NE import Data.Map (Map) import Data.Map qualified as Map -import Data.Maybe (catMaybes, isNothing) +import Data.Maybe (catMaybes, isNothing, mapMaybe, maybeToList) import Data.String.Conversion (toText) import Data.Text (Text) import Data.Traversable (for) import Data.Tree (Tree (..)) import DepTypes (DepType (MavenType)) -import Discovery.Walk (findFileInAncestor) +import Discovery.Walk (WalkStep (WalkContinue), findFileInAncestor, findFileNamed, walk') import Effect.Exec ( AllowErr (Never), CandidateAnalysisCommands (..), @@ -133,6 +139,11 @@ execPluginReactor projectdir outputdir plugin = do cmd <- mavenPluginReactorCmd projectdir outputdir plugin execPlugin (const cmd) projectdir plugin +execPluginVerboseGraph :: (CandidateCommandEffs sig m, Has ReadFS sig m) => Path Abs Dir -> DepGraphPlugin -> m () +execPluginVerboseGraph dir plugin = do + cmd <- mavenPluginVerboseGraphCmd dir plugin + execPlugin (const cmd) dir plugin + outputFile :: Path Rel File outputFile = $(mkRelFile "target/dependency-graph.txt") @@ -146,6 +157,16 @@ reactorOutputFilename = $(mkRelFile "fossa-reactor-graph.json") parseReactorOutput :: (Has ReadFS sig m, Has Diagnostics sig m) => (Path Abs Dir) -> m ReactorOutput parseReactorOutput dir = readContentsJson $ dir reactorOutputFilename +verboseGraphFileName :: String +verboseGraphFileName = "fossa-depgraph-verbose.json" + +-- | Find and parse the output of 'execPluginVerboseGraph'. The @graph@ goal +-- runs per reactor module, writing into each module's build directory. +parseVerboseGraphs :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> m [VerboseGraph] +parseVerboseGraphs dir = do + outputs <- walk' (\_ _ files -> pure (maybeToList (findFileNamed verboseGraphFileName files), WalkContinue)) dir + traverse readContentsJson outputs + textArtifactToPluginOutput :: Has Diagnostics sig m => Tree TextArtifact -> m PluginOutput textArtifactToPluginOutput ta = buildPluginOutput ta @@ -262,6 +283,27 @@ mavenPluginDependenciesCmd workdir plugin = do "-DrepeatTransitiveDependenciesInTextGraph=true" ] +-- | The @aggregate@ goal above cannot report edges Maven resolved away as +-- duplicates (it hardcodes @NodeResolution.INCLUDED@ and has no +-- @showDuplicates@ option), so a package shared by several parents appears +-- under only one of them. The non-aggregating +-- [graph goal](https://ferstl.github.io/depgraph-maven-plugin/graph-mojo.html) +-- with @showDuplicates@ reports those edges as @OMITTED_FOR_DUPLICATE@. +mavenPluginVerboseGraphCmd :: (CandidateCommandEffs sig m, Has ReadFS sig m) => Path Abs Dir -> DepGraphPlugin -> m Command +mavenPluginVerboseGraphCmd workdir plugin = do + candidates <- mavenCmdCandidates workdir + mkAnalysisCommand candidates workdir args Never + where + args = + [ group plugin <> ":" <> artifact plugin <> ":" <> version plugin <> ":graph" + , "-DgraphFormat=json" + , -- match the node identity used by the aggregate command + "-DmergeScopes" + , -- request Maven's verbose graph so duplicate-resolved edges are reported + "-DshowDuplicates=true" + , "-DoutputFileName=" <> toText verboseGraphFileName + ] + -- | The reactor command is documented -- [here.](https://ferstl.github.io/depgraph-maven-plugin/reactor-mojo.html) -- We set outputDirectory explicitly so that the file is written to a known spot even if the pom file @@ -329,3 +371,86 @@ data Edge = Edge , edgeTo :: Int } deriving (Eq, Ord, Show) + +-- | The depgraph plugin's JSON format. Edges join to artifacts via the string +-- @id@ field; the numeric ids use two unrelated counters and cannot be used. +data VerboseGraph = VerboseGraph + { verboseArtifacts :: [VerboseArtifact] + , verboseEdges :: [VerboseEdge] + } + deriving (Eq, Ord, Show) + +instance FromJSON VerboseGraph where + parseJSON = withObject "Verbose graph" $ + \o -> + VerboseGraph + <$> (o .:? "artifacts" .!= []) + <*> (o .:? "dependencies" .!= []) + +data VerboseArtifact = VerboseArtifact + { verboseArtifactId :: Text + , verboseArtifactGroupId :: Text + , verboseArtifactArtifactId :: Text + , verboseArtifactVersion :: Text + } + deriving (Eq, Ord, Show) + +instance FromJSON VerboseArtifact where + parseJSON = withObject "Verbose graph artifact" $ + \o -> + VerboseArtifact + <$> o .: "id" + <*> o .: "groupId" + <*> o .: "artifactId" + <*> o .: "version" + +data VerboseEdge = VerboseEdge + { verboseEdgeFrom :: Text + , verboseEdgeTo :: Text + , verboseEdgeResolution :: Text + } + deriving (Eq, Ord, Show) + +instance FromJSON VerboseEdge where + parseJSON = withObject "Verbose graph dependency" $ + \o -> + VerboseEdge + <$> o .: "from" + <*> o .: "to" + <*> (o .:? "resolution" .!= "INCLUDED") + +-- | Merge @OMITTED_FOR_DUPLICATE@ edges into the aggregate output; both +-- endpoints of a duplicate already exist there. @OMITTED_FOR_CONFLICT@ edges +-- point at a losing version the build does not ship, so they are skipped. +augmentWithDuplicateEdges :: PluginOutput -> [VerboseGraph] -> PluginOutput +augmentWithDuplicateEdges output@PluginOutput{outArtifacts, outEdges} verboseGraphs = + output{outEdges = nub (outEdges <> concatMap duplicateEdges verboseGraphs)} + where + idByGav :: Map (Text, Text, Text) Int + idByGav = + Map.fromList $ + map (\a -> ((artifactGroupId a, artifactArtifactId a, artifactVersion a), artifactNumericId a)) outArtifacts + + duplicateEdges :: VerboseGraph -> [Edge] + duplicateEdges VerboseGraph{verboseArtifacts, verboseEdges} = mapMaybe toAggregateEdge verboseEdges + where + gavByVerboseId :: Map Text (Text, Text, Text) + gavByVerboseId = + Map.fromList $ + map + (\a -> (verboseArtifactId a, (verboseArtifactGroupId a, verboseArtifactArtifactId a, verboseArtifactVersion a))) + verboseArtifacts + + lookupAggregateId :: Text -> Maybe Int + lookupAggregateId verboseId = Map.lookup verboseId gavByVerboseId >>= (`Map.lookup` idByGav) + + toAggregateEdge :: VerboseEdge -> Maybe Edge + toAggregateEdge VerboseEdge{verboseEdgeFrom, verboseEdgeTo, verboseEdgeResolution} = + if verboseEdgeResolution /= "OMITTED_FOR_DUPLICATE" + then Nothing + else case (lookupAggregateId verboseEdgeFrom, lookupAggregateId verboseEdgeTo) of + (Just parent, Just child) -> + if parent == child + then Nothing + else Just (Edge parent child) + _ -> Nothing diff --git a/src/Strategy/Maven/PluginStrategy.hs b/src/Strategy/Maven/PluginStrategy.hs index d146bdeb96..c0240448a1 100644 --- a/src/Strategy/Maven/PluginStrategy.hs +++ b/src/Strategy/Maven/PluginStrategy.hs @@ -11,6 +11,7 @@ import Control.Effect.Diagnostics ( ToDiagnostic (renderDiagnostic), context, errCtx, + recover, warnOnErr, (<||>), ) @@ -20,6 +21,7 @@ import Control.Monad (when) import Data.Foldable (traverse_) import Data.Map.Strict (Map) import Data.Map.Strict qualified as Map +import Data.Maybe (fromMaybe) import Data.Set qualified as Set import Data.Text (Text) import DepTypes ( @@ -42,13 +44,16 @@ import Strategy.Maven.Plugin ( Edge (..), PluginOutput (..), ReactorOutput (ReactorOutput), + augmentWithDuplicateEdges, depGraphPlugin, depGraphPluginLegacy, execPluginAggregate, execPluginReactor, + execPluginVerboseGraph, installPlugin, parsePluginOutput, parseReactorOutput, + parseVerboseGraphs, reactorArtifactName, reactorArtifacts, withUnpackedPlugin, @@ -103,9 +108,30 @@ analyze dir plugin = do errCtx MvnPluginExecFailed $ execPluginAggregate dir plugin pluginOutput <- parsePluginOutput dir - context "Building dependency graph" $ pure (buildGraph reactorOutput pluginOutput) + pluginOutput' <- recoverDuplicateEdges dir plugin pluginOutput + context "Building dependency graph" $ pure (buildGraph reactorOutput pluginOutput') pure (graph, Complete) +-- | Recover edges Maven resolved away as duplicates (see +-- 'Strategy.Maven.Plugin.mavenPluginVerboseGraphCmd') and merge them into the +-- parsed output. Best-effort: on failure the aggregate output is used as-is. +recoverDuplicateEdges :: + ( CandidateCommandEffs sig m + , Has ReadFS sig m + ) => + Path Abs Dir -> + DepGraphPlugin -> + PluginOutput -> + m PluginOutput +recoverDuplicateEdges dir plugin pluginOutput = + context "Running plugin to recover duplicate-resolved edges" $ do + recovered <- + recover $ + warnOnErr DuplicateEdgesNotRecovered $ do + execPluginVerboseGraph dir plugin + augmentWithDuplicateEdges pluginOutput <$> parseVerboseGraphs dir + pure (fromMaybe pluginOutput recovered) + data MvnPluginInstallFailed = MvnPluginInstallFailed instance ToDiagnostic MvnPluginInstallFailed where renderDiagnostic (MvnPluginInstallFailed) = do @@ -118,6 +144,12 @@ instance ToDiagnostic MvnPluginExecFailed where let header = "Failed to execute maven plugin for analysis" Errata (Just header) [] Nothing +data DuplicateEdgesNotRecovered = DuplicateEdgesNotRecovered +instance ToDiagnostic DuplicateEdgesNotRecovered where + renderDiagnostic DuplicateEdgesNotRecovered = do + let header = "Failed to recover duplicate-resolved dependency edges; transitive dependencies shared between multiple parents may be attributed to only one of them." + Errata (Just header) [] Nothing + data MayIncludeSubmodule = MayIncludeSubmodule instance ToDiagnostic MayIncludeSubmodule where renderDiagnostic MayIncludeSubmodule = do diff --git a/test/Maven/PluginSpec.hs b/test/Maven/PluginSpec.hs index 58b9be30f8..a3508d5bf8 100644 --- a/test/Maven/PluginSpec.hs +++ b/test/Maven/PluginSpec.hs @@ -2,18 +2,31 @@ module Maven.PluginSpec (spec) where +import Data.Aeson (eitherDecode) +import Data.ByteString.Lazy.Char8 qualified as BS import Data.Text (Text) import Data.Tree (Tree (..)) -import Strategy.Maven.Plugin (Artifact (..), Edge (..), PluginOutput (..), textArtifactToPluginOutput) +import Strategy.Maven.Plugin ( + Artifact (..), + Edge (..), + PluginOutput (..), + VerboseArtifact (..), + VerboseEdge (..), + VerboseGraph (..), + augmentWithDuplicateEdges, + textArtifactToPluginOutput, + ) import Strategy.Maven.PluginTree (TextArtifact (..), parseTextArtifact) import Test.Effect (expectationFailure', it', shouldBe', shouldContain', shouldMatchList') -import Test.Hspec (Spec, describe) +import Test.Hspec (Spec, describe, it, shouldBe) import Text.Megaparsec (parseMaybe) import Text.RawString.QQ (r) spec :: Spec spec = do textArtifactConversionSpec + verboseGraphParsingSpec + augmentWithDuplicateEdgesSpec singleTextArtifact :: TextArtifact singleTextArtifact = @@ -201,3 +214,102 @@ kafkaClientCompile = Artifact 6 "org.apache.kafka" "kafka-clients" "3.0.2" ["com kafkaClientTest :: Artifact kafkaClientTest = kafkaClientCompile{artifactNumericId = 1, artifactScopes = ["test"]} + +-- | depgraph JSON with @showDuplicates=true@. The numeric ids deliberately do +-- not line up: depgraph numbers artifacts and edge endpoints independently, so +-- edges must join to artifacts via the string ids. +verboseGraphJson :: BS.ByteString +verboseGraphJson = + BS.pack + [r|{ + "graphName": "example", + "artifacts": [ + { "id": "org.example:app:jar", "numericId": 7, "groupId": "org.example", "artifactId": "app", "version": "1.0.0", "scopes": [], "types": ["jar"] }, + { "id": "org.apache.poi:poi-ooxml:jar", "numericId": 3, "groupId": "org.apache.poi", "artifactId": "poi-ooxml", "version": "5.2.5", "scopes": ["compile"], "types": ["jar"] }, + { "id": "org.apache.logging.log4j:log4j-api:jar", "numericId": 9, "groupId": "org.apache.logging.log4j", "artifactId": "log4j-api", "version": "2.21.1", "scopes": ["compile"], "types": ["jar"] }, + { "id": "org.apache.logging.log4j:log4j-core:jar", "numericId": 1, "groupId": "org.apache.logging.log4j", "artifactId": "log4j-core", "version": "2.21.1", "scopes": ["compile"], "types": ["jar"] } + ], + "dependencies": [ + { "from": "org.example:app:jar", "to": "org.apache.poi:poi-ooxml:jar", "numericFrom": 1, "numericTo": 2, "resolution": "INCLUDED" }, + { "from": "org.example:app:jar", "to": "org.apache.logging.log4j:log4j-core:jar", "numericFrom": 1, "numericTo": 4, "resolution": "INCLUDED" }, + { "from": "org.apache.poi:poi-ooxml:jar", "to": "org.apache.logging.log4j:log4j-api:jar", "numericFrom": 2, "numericTo": 3, "resolution": "INCLUDED" }, + { "from": "org.apache.logging.log4j:log4j-core:jar", "to": "org.apache.logging.log4j:log4j-api:jar", "numericFrom": 4, "numericTo": 3, "resolution": "OMITTED_FOR_DUPLICATE" } + ] +}|] + +expectedVerboseGraph :: VerboseGraph +expectedVerboseGraph = + VerboseGraph + { verboseArtifacts = + [ VerboseArtifact "org.example:app:jar" "org.example" "app" "1.0.0" + , VerboseArtifact "org.apache.poi:poi-ooxml:jar" "org.apache.poi" "poi-ooxml" "5.2.5" + , VerboseArtifact "org.apache.logging.log4j:log4j-api:jar" "org.apache.logging.log4j" "log4j-api" "2.21.1" + , VerboseArtifact "org.apache.logging.log4j:log4j-core:jar" "org.apache.logging.log4j" "log4j-core" "2.21.1" + ] + , verboseEdges = + [ VerboseEdge "org.example:app:jar" "org.apache.poi:poi-ooxml:jar" "INCLUDED" + , VerboseEdge "org.example:app:jar" "org.apache.logging.log4j:log4j-core:jar" "INCLUDED" + , VerboseEdge "org.apache.poi:poi-ooxml:jar" "org.apache.logging.log4j:log4j-api:jar" "INCLUDED" + , VerboseEdge "org.apache.logging.log4j:log4j-core:jar" "org.apache.logging.log4j:log4j-api:jar" "OMITTED_FOR_DUPLICATE" + ] + } + +verboseGraphParsingSpec :: Spec +verboseGraphParsingSpec = + describe "verbose graph parsing" $ + it "should parse the depgraph plugin's json format" $ + eitherDecode verboseGraphJson `shouldBe` Right expectedVerboseGraph + +-- Aggregate output for the same build, with its own unrelated numeric ids. +mkAggregateArtifact :: Int -> Text -> Text -> Text -> Bool -> Artifact +mkAggregateArtifact numericId groupId artifactId version isDirect = + Artifact + { artifactNumericId = numericId + , artifactGroupId = groupId + , artifactArtifactId = artifactId + , artifactVersion = version + , artifactScopes = ["compile"] + , artifactOptional = False + , artifactIsDirect = isDirect + } + +aggregateOutput :: PluginOutput +aggregateOutput = + PluginOutput + { outArtifacts = + [ mkAggregateArtifact 10 "org.apache.poi" "poi-ooxml" "5.2.5" True + , mkAggregateArtifact 11 "org.apache.logging.log4j" "log4j-api" "2.21.1" False + , mkAggregateArtifact 12 "org.apache.logging.log4j" "log4j-core" "2.21.1" True + ] + , outEdges = [Edge 10 11] + } + +augmentWithDuplicateEdgesSpec :: Spec +augmentWithDuplicateEdgesSpec = + describe "augmentWithDuplicateEdges" $ do + it "should add duplicate-resolved edges between existing artifacts" $ + outEdges (augmentWithDuplicateEdges aggregateOutput [expectedVerboseGraph]) + `shouldBe` [Edge 10 11, Edge 12 11] + + it "should not modify artifacts" $ + outArtifacts (augmentWithDuplicateEdges aggregateOutput [expectedVerboseGraph]) + `shouldBe` outArtifacts aggregateOutput + + it "should ignore included edges and artifacts absent from the aggregate output" $ do + let onlyIncluded = + expectedVerboseGraph + { verboseEdges = + [ VerboseEdge "org.example:app:jar" "org.apache.poi:poi-ooxml:jar" "INCLUDED" + , VerboseEdge "org.apache.poi:poi-ooxml:jar" "org.apache.logging.log4j:log4j-api:jar" "INCLUDED" + ] + } + outEdges (augmentWithDuplicateEdges aggregateOutput [onlyIncluded]) + `shouldBe` outEdges aggregateOutput + + it "should not duplicate an edge that already exists" $ do + let alreadyPresent = aggregateOutput{outEdges = [Edge 10 11, Edge 12 11]} + outEdges (augmentWithDuplicateEdges alreadyPresent [expectedVerboseGraph]) + `shouldBe` [Edge 10 11, Edge 12 11] + + it "should do nothing without verbose graphs" $ + augmentWithDuplicateEdges aggregateOutput [] `shouldBe` aggregateOutput