Build flat dependency trees while auditing Gradle projects (#976)

* Upgraded to the new gradle-dep-tree version, to make the scan faster and more memory-efficient.

buildscripts/download-jars.sh

@@ -7,7 +7,7 @@
 # https://github.com/jfrog/maven-dep-tree
 
 # Once you have updated the versions mentioned below, please execute this script from the root directory of the jfrog-cli-core to ensure the JAR files are updated.
-GRADLE_DEP_TREE_VERSION="2.2.0"
+GRADLE_DEP_TREE_VERSION="3.0.0"
 MAVEN_DEP_TREE_VERSION="1.0.0"
 
 curl -fL https://releases.jfrog.io/artifactory/oss-release-local/com/jfrog/gradle-dep-tree/${GRADLE_DEP_TREE_VERSION}/gradle-dep-tree-${GRADLE_DEP_TREE_VERSION}.jar -o xray/commands/audit/sca/java/gradle-dep-tree.jar

xray/commands/audit/sca/java/gradle.go

@@ -2,11 +2,8 @@ package java
 
 import (
 	_ "embed"
-	"encoding/base64"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/jfrog/gofrog/datastructures"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -58,60 +55,12 @@ allprojects {
 var gradleDepTreeJar []byte
 
 type depTreeManager struct {
-	dependenciesTree
 	server       *config.ServerDetails
 	releasesRepo string
 	depsRepo     string
 	useWrapper   bool
 }
 
-// dependenciesTree represents a map between dependencies to their children dependencies in multiple projects.
-type dependenciesTree struct {
-	tree map[string][]dependenciesPaths
-}
-
-// dependenciesPaths represents a map between dependencies to their children dependencies in a single project.
-type dependenciesPaths struct {
-	Paths map[string]dependenciesPaths `json:"children"`
-}
-
-// The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project.
-// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in dependenciesTree struct.
-func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error {
-	outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n")
-	for _, path := range outputFiles {
-		tree, err := os.ReadFile(strings.TrimSpace(path))
-		if err != nil {
-			return errorutils.CheckError(err)
-		}
-
-		encodedFileName := path[strings.LastIndex(path, string(os.PathSeparator))+1:]
-		decodedFileName, err := base64.StdEncoding.DecodeString(encodedFileName)
-		if err != nil {
-			return errorutils.CheckError(err)
-		}
-
-		if err = dtp.appendDependenciesPaths(tree, string(decodedFileName)); err != nil {
-			return errorutils.CheckError(err)
-		}
-	}
-	return nil
-}
-
-func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte, fileName string) error {
-	var deps dependenciesPaths
-	if err := json.Unmarshal(jsonDepTree, &deps); err != nil {
-		return errorutils.CheckError(err)
-	}
-	if dtp.tree == nil {
-		dtp.tree = make(map[string][]dependenciesPaths)
-	}
-	if len(deps.Paths) > 0 {
-		dtp.tree[fileName] = append(dtp.tree[fileName], deps)
-	}
-	return nil
-}
-
 func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, err error) {
 	manager := &depTreeManager{useWrapper: params.UseWrapper}
 	if params.IgnoreConfigFile {
@@ -130,7 +79,7 @@ func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []*
 	if err != nil {
 		return
 	}
-	dependencyTree, uniqueDeps, err = manager.getGraphFromDepTree(outputFileContent)
+	dependencyTree, uniqueDeps, err = getGraphFromDepTree(outputFileContent)
 	return
 }
 
@@ -163,7 +112,7 @@ func (dtp *depTreeManager) createDepTreeScriptAndGetDir() (tmpDir string, err er
 	if err != nil {
 		return
 	}
-	gradleDepTreeJarPath := filepath.Join(tmpDir, string(gradleDepTreeJarFile))
+	gradleDepTreeJarPath := filepath.Join(tmpDir, gradleDepTreeJarFile)
 	if err = errorutils.CheckError(os.WriteFile(gradleDepTreeJarPath, gradleDepTreeJar, 0666)); err != nil {
 		return
 	}
@@ -237,42 +186,6 @@ func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileConte
 	return
 }
 
-// Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input
-func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*xrayUtils.GraphNode, []string, error) {
-	if err := dtp.parseDepTreeFiles(outputFileContent); err != nil {
-		return nil, nil, err
-	}
-	var depsGraph []*xrayUtils.GraphNode
-	uniqueDepsSet := datastructures.MakeSet[string]()
-	for dependency, children := range dtp.tree {
-		directDependency := &xrayUtils.GraphNode{
-			Id:    GavPackageTypeIdentifier + dependency,
-			Nodes: []*xrayUtils.GraphNode{},
-		}
-		for _, childPath := range children {
-			populateGradleDependencyTree(directDependency, childPath, uniqueDepsSet)
-		}
-		depsGraph = append(depsGraph, directDependency)
-	}
-	return depsGraph, uniqueDepsSet.ToSlice(), nil
-}
-
-func populateGradleDependencyTree(currNode *xrayUtils.GraphNode, currNodeChildren dependenciesPaths, uniqueDepsSet *datastructures.Set[string]) {
-	uniqueDepsSet.Add(currNode.Id)
-	for gav, children := range currNodeChildren.Paths {
-		childNode := &xrayUtils.GraphNode{
-			Id:     GavPackageTypeIdentifier + gav,
-			Nodes:  []*xrayUtils.GraphNode{},
-			Parent: currNode,
-		}
-		if currNode.NodeHasLoop() {
-			return
-		}
-		populateGradleDependencyTree(childNode, children, uniqueDepsSet)
-		currNode.Nodes = append(currNode.Nodes, childNode)
-	}
-}
-
 func getDepTreeArtifactoryRepository(remoteRepo string, server *config.ServerDetails) (string, error) {
 	if remoteRepo == "" || server.IsEmpty() {
 		return "", nil

xray/commands/audit/sca/java/gradle_test.go

@@ -47,10 +47,10 @@ func TestGradleTreesWithoutConfig(t *testing.T) {
 	// Run getModulesDependencyTrees
 	modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{})
 	if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) {
-		assert.Len(t, uniqueDeps, 11)
-		assert.Len(t, modulesDependencyTrees, 2)
+		assert.Len(t, uniqueDeps, 9)
+		assert.Len(t, modulesDependencyTrees, 5)
 		// Check module
-		module := sca.GetAndAssertNode(t, modulesDependencyTrees, "webservice")
+		module := sca.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.example.gradle:webservice:1.0")
 		assert.Len(t, module.Nodes, 7)
 
 		// Check direct dependency
@@ -71,10 +71,10 @@ func TestGradleTreesWithConfig(t *testing.T) {
 	// Run getModulesDependencyTrees
 	modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true})
 	if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) {
-		assert.Len(t, modulesDependencyTrees, 3)
-		assert.Len(t, uniqueDeps, 11)
+		assert.Len(t, modulesDependencyTrees, 5)
+		assert.Len(t, uniqueDeps, 8)
 		// Check module
-		module := sca.GetAndAssertNode(t, modulesDependencyTrees, "api")
+		module := sca.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test.gradle.publish:api:1.0-SNAPSHOT")
 		assert.Len(t, module.Nodes, 4)
 
 		// Check direct dependency
@@ -86,22 +86,6 @@ func TestGradleTreesWithConfig(t *testing.T) {
 	}
 }
 
-func TestGradleTreesExcludeTestDeps(t *testing.T) {
-	// Create and change directory to test workspace
-	tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server")
-	defer cleanUp()
-	assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700))
-
-	// Run getModulesDependencyTrees
-	modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true})
-	if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) {
-		assert.Len(t, modulesDependencyTrees, 2)
-		assert.Len(t, uniqueDeps, 11)
-		// Check direct dependency
-		assert.Nil(t, sca.GetModule(modulesDependencyTrees, "services"))
-	}
-}
-
 func TestIsGradleWrapperExist(t *testing.T) {
 	// Check Gradle wrapper doesn't exist
 	isWrapperExist, err := isGradleWrapperExist()
@@ -168,67 +152,6 @@ func TestGetDepTreeArtifactoryRepository(t *testing.T) {
 	}
 }
 
-func TestGetGraphFromDepTree(t *testing.T) {
-	// Create and change directory to test workspace
-	tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server")
-	defer func() {
-		cleanUp()
-	}()
-	assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700))
-	testCase := struct {
-		name               string
-		expectedTree       map[string]map[string]string
-		expectedUniqueDeps []string
-	}{
-		name: "ValidOutputFileContent",
-		expectedTree: map[string]map[string]string{
-			GavPackageTypeIdentifier + "shared":                   {},
-			GavPackageTypeIdentifier + filepath.Base(tempDirPath): {},
-			GavPackageTypeIdentifier + "services":                 {},
-			GavPackageTypeIdentifier + "webservice": {
-				GavPackageTypeIdentifier + "junit:junit:4.11":                            "",
-				GavPackageTypeIdentifier + "commons-io:commons-io:1.2":                   "",
-				GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7":              "",
-				GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0":         "",
-				GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0":            "",
-				GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4":               "",
-				GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": "",
-			},
-			GavPackageTypeIdentifier + "api": {
-				GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7":      "",
-				GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "",
-				GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4":       "",
-			},
-		},
-		expectedUniqueDeps: []string{
-			GavPackageTypeIdentifier + "webservice",
-			GavPackageTypeIdentifier + "junit:junit:4.11",
-			GavPackageTypeIdentifier + "commons-io:commons-io:1.2",
-			GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7",
-			GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0",
-			GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0",
-			GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2",
-			GavPackageTypeIdentifier + "api",
-			GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4",
-			GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3",
-			GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2",
-		},
-	}
-
-	manager := &depTreeManager{}
-	outputFileContent, err := manager.runGradleDepTree()
-	assert.NoError(t, err)
-	depTree, uniqueDeps, err := (&depTreeManager{}).getGraphFromDepTree(outputFileContent)
-	assert.NoError(t, err)
-	assert.ElementsMatch(t, uniqueDeps, testCase.expectedUniqueDeps, "First is actual, Second is Expected")
-
-	for _, dependency := range depTree {
-		depChild, exists := testCase.expectedTree[dependency.Id]
-		assert.True(t, exists)
-		assert.Equal(t, len(depChild), len(dependency.Nodes))
-	}
-}
-
 func TestCreateDepTreeScript(t *testing.T) {
 	manager := &depTreeManager{}
 	tmpDir, err := manager.createDepTreeScriptAndGetDir()

xray/commands/audit/sca/java/javautils.go

@@ -1,12 +1,15 @@
 package java
 
 import (
+	"encoding/json"
 	"github.com/jfrog/gofrog/datastructures"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils"
 	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	buildinfo "github.com/jfrog/build-info-go/entities"
@@ -58,7 +61,7 @@ func createGavDependencyTree(buildConfig *artifactoryUtils.BuildConfiguration) (
 	if len(generatedBuildsInfos) == 0 {
 		return nil, nil, errorutils.CheckErrorf("Couldn't find build " + buildName + "/" + buildNumber)
 	}
-	modules := []*xrayUtils.GraphNode{}
+	var modules []*xrayUtils.GraphNode
 	uniqueDepsSet := datastructures.MakeSet[string]()
 	for _, module := range generatedBuildsInfos[0].Modules {
 		modules = append(modules, addModuleTree(module, uniqueDepsSet))
@@ -173,3 +176,73 @@ func (dm *dependencyMultimap) putChild(parent string, child *buildinfo.Dependenc
 func (dm *dependencyMultimap) getChildren(parent string) map[string]*buildinfo.Dependency {
 	return dm.multimap[parent]
 }
+
+// The structure of a dependency tree of a module in a Gradle/Maven project, as created by the gradle-dep-tree and maven-dep-tree plugins.
+type moduleDepTree struct {
+	Root  string                 `json:"root"`
+	Nodes map[string]depTreeNode `json:"nodes"`
+}
+
+type depTreeNode struct {
+	Children []string `json:"children"`
+}
+
+// getGraphFromDepTree reads the output files of the gradle-dep-tree and maven-dep-tree plugins and returns them as a slice of GraphNodes.
+// It takes the output of the plugin's run (which is a byte representation of a list of paths of the output files, separated by newlines) as input.
+func getGraphFromDepTree(depTreeOutput []byte) (depsGraph []*xrayUtils.GraphNode, uniqueDeps []string, err error) {
+	modules, err := parseDepTreeFiles(depTreeOutput)
+	if err != nil {
+		return
+	}
+	uniqueDepsSet := datastructures.MakeSet[string]()
+	for _, moduleTree := range modules {
+		directDependency := &xrayUtils.GraphNode{
+			Id:    GavPackageTypeIdentifier + moduleTree.Root,
+			Nodes: []*xrayUtils.GraphNode{},
+		}
+		populateDependencyTree(directDependency, moduleTree.Root, moduleTree, uniqueDepsSet)
+		depsGraph = append(depsGraph, directDependency)
+	}
+	uniqueDeps = uniqueDepsSet.ToSlice()
+	return
+}
+
+func populateDependencyTree(currNode *xrayUtils.GraphNode, currNodeId string, moduleTree *moduleDepTree, uniqueDepsSet *datastructures.Set[string]) {
+	if currNode.NodeHasLoop() {
+		return
+	}
+	for _, childId := range moduleTree.Nodes[currNodeId].Children {
+		childGav := GavPackageTypeIdentifier + childId
+		childNode := &xrayUtils.GraphNode{
+			Id:     childGav,
+			Nodes:  []*xrayUtils.GraphNode{},
+			Parent: currNode,
+		}
+		uniqueDepsSet.Add(childGav)
+		populateDependencyTree(childNode, childId, moduleTree, uniqueDepsSet)
+		currNode.Nodes = append(currNode.Nodes, childNode)
+	}
+}
+
+func parseDepTreeFiles(jsonFilePaths []byte) ([]*moduleDepTree, error) {
+	outputFilePaths := strings.Split(strings.TrimSpace(string(jsonFilePaths)), "\n")
+	var modules []*moduleDepTree
+	for _, path := range outputFilePaths {
+		results, err := parseDepTreeFile(path)
+		if err != nil {
+			return nil, err
+		}
+		modules = append(modules, results)
+	}
+	return modules, nil
+}
+
+func parseDepTreeFile(path string) (results *moduleDepTree, err error) {
+	depTreeJson, err := os.ReadFile(strings.TrimSpace(path))
+	if errorutils.CheckError(err) != nil {
+		return
+	}
+	results = &moduleDepTree{}
+	err = errorutils.CheckError(json.Unmarshal(depTreeJson, &results))
+	return
+}