Fix Python Contextual analysis missing dependencies issue

xray/audit/jas/applicabilitymanager.go

@@ -7,9 +7,9 @@ import (
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 	"github.com/jfrog/jfrog-client-go/xray/services"
-	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
 	"github.com/owenrumney/go-sarif/v2/sarif"
 	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 	"strings"
 )
 
@@ -27,9 +27,9 @@ const (
 // map[string]string: A map containing the applicability result of each XRAY CVE.
 // bool: true if the user is entitled to the applicability scan, false otherwise.
 // error: An error object (if any).
-func getApplicabilityScanResults(xrayResults []services.ScanResponse, dependencyTrees []*xrayUtils.GraphNode,
+func getApplicabilityScanResults(xrayResults []services.ScanResponse, directDependencies []string,
 	scannedTechnologies []coreutils.Technology, scanner *AdvancedSecurityScanner) (results map[string]string, err error) {
-	applicabilityScanManager := newApplicabilityScanManager(xrayResults, dependencyTrees, scanner)
+	applicabilityScanManager := newApplicabilityScanManager(xrayResults, directDependencies, scanner)
 	if !applicabilityScanManager.shouldRunApplicabilityScan(scannedTechnologies) {
 		log.Debug("The technologies that have been scanned are currently not supported for contextual analysis scanning, or we couldn't find any vulnerable direct dependencies. Skipping....")
 		return
@@ -49,8 +49,7 @@ type ApplicabilityScanManager struct {
 	scanner                  *AdvancedSecurityScanner
 }
 
-func newApplicabilityScanManager(xrayScanResults []services.ScanResponse, dependencyTrees []*xrayUtils.GraphNode, scanner *AdvancedSecurityScanner) (manager *ApplicabilityScanManager) {
-	directDependencies := getDirectDependenciesSet(dependencyTrees)
+func newApplicabilityScanManager(xrayScanResults []services.ScanResponse, directDependencies []string, scanner *AdvancedSecurityScanner) (manager *ApplicabilityScanManager) {
 	directDependenciesCves := extractDirectDependenciesCvesFromScan(xrayScanResults, directDependencies)
 	return &ApplicabilityScanManager{
 		applicabilityScanResults: map[string]string{},
@@ -62,20 +61,24 @@ func newApplicabilityScanManager(xrayScanResults []services.ScanResponse, depend
 
 // This function gets a list of xray scan responses that contain direct and indirect vulnerabilities and returns only direct
 // vulnerabilities of the scanned project, ignoring indirect vulnerabilities
-func extractDirectDependenciesCvesFromScan(xrayScanResults []services.ScanResponse, directDependencies *datastructures.Set[string]) *datastructures.Set[string] {
+func extractDirectDependenciesCvesFromScan(xrayScanResults []services.ScanResponse, directDependencies []string) *datastructures.Set[string] {
 	directsCves := datastructures.MakeSet[string]()
 	for _, scanResult := range xrayScanResults {
 		for _, vulnerability := range scanResult.Vulnerabilities {
 			if isDirectComponents(maps.Keys(vulnerability.Components), directDependencies) {
 				for _, cve := range vulnerability.Cves {
-					directsCves.Add(cve.Id)
+					if cve.Id != "" {
+						directsCves.Add(cve.Id)
+					}
 				}
 			}
 		}
 		for _, violation := range scanResult.Violations {
 			if isDirectComponents(maps.Keys(violation.Components), directDependencies) {
 				for _, cve := range violation.Cves {
-					directsCves.Add(cve.Id)
+					if cve.Id != "" {
+						directsCves.Add(cve.Id)
+					}
 				}
 			}
 		}
@@ -84,26 +87,15 @@ func extractDirectDependenciesCvesFromScan(xrayScanResults []services.ScanRespon
 	return directsCves
 }
 
-func isDirectComponents(components []string, directDependencies *datastructures.Set[string]) bool {
+func isDirectComponents(components []string, directDependencies []string) bool {
 	for _, component := range components {
-		if directDependencies.Exists(component) {
+		if slices.Contains(directDependencies, component) {
 			return true
 		}
 	}
 	return false
 }
 
-// This function retrieves the dependency trees of the scanned project and extracts a set that contains only the direct dependencies.
-func getDirectDependenciesSet(dependencyTrees []*xrayUtils.GraphNode) *datastructures.Set[string] {
-	directDependencies := datastructures.MakeSet[string]()
-	for _, tree := range dependencyTrees {
-		for _, node := range tree.Nodes {
-			directDependencies.Add(node.Id)
-		}
-	}
-	return directDependencies
-}
-
 func (a *ApplicabilityScanManager) Run(wd string) (err error) {
 	if len(a.scanner.workingDirs) > 1 {
 		log.Info("Running applicability scanning in the", wd, "directory...")

xray/audit/jas/applicabilitymanager_test.go

@@ -13,6 +13,9 @@ import (
 	"testing"
 )
 
+var fakeDirectDeps = []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}
+var fakeMultiDirectDeps = []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency", "issueId_3_direct_dependency", "issueId_4_direct_dependency"}
+
 func TestNewApplicabilityScanManager_InputIsValid(t *testing.T) {
 	// Act
 	assert.NoError(t, rtutils.DownloadAnalyzerManagerIfNeeded())
@@ -23,7 +26,7 @@ func TestNewApplicabilityScanManager_InputIsValid(t *testing.T) {
 			assert.NoError(t, scanner.scannerDirCleanupFunc())
 		}
 	}()
-	applicabilityManager := newApplicabilityScanManager(fakeBasicXrayResults, fakeBasicDependencyGraph, scanner)
+	applicabilityManager := newApplicabilityScanManager(fakeBasicXrayResults, fakeDirectDeps, scanner)
 
 	// Assert
 	assert.NotEmpty(t, applicabilityManager)
@@ -84,7 +87,7 @@ func TestNewApplicabilityScanManager_NoDirectDependenciesInScan(t *testing.T) {
 			assert.NoError(t, scanner.scannerDirCleanupFunc())
 		}
 	}()
-	applicabilityManager := newApplicabilityScanManager(noDirectDependenciesResults, fakeBasicDependencyGraph, scanner)
+	applicabilityManager := newApplicabilityScanManager(noDirectDependenciesResults, fakeDirectDeps, scanner)
 
 	// Assert
 	assert.NotEmpty(t, applicabilityManager)
@@ -293,43 +296,6 @@ func TestExtractXrayDirectVulnerabilities(t *testing.T) {
 	}
 }
 
-func TestGetDirectDependenciesList(t *testing.T) {
-	tests := []struct {
-		dependenciesTrees []*xrayUtils.GraphNode
-		expectedResult    []string
-	}{
-		{
-			dependenciesTrees: nil,
-			expectedResult:    []string{},
-		},
-		{
-			dependenciesTrees: []*xrayUtils.GraphNode{
-				{Id: "parent_node_id", Nodes: []*xrayUtils.GraphNode{
-					{Id: "issueId_1_direct_dependency", Nodes: []*xrayUtils.GraphNode{{Id: "issueId_1_non_direct_dependency"}}},
-					{Id: "issueId_2_direct_dependency", Nodes: nil},
-				},
-				},
-			},
-			expectedResult: []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"},
-		},
-		{
-			dependenciesTrees: []*xrayUtils.GraphNode{
-				{Id: "parent_node_id", Nodes: []*xrayUtils.GraphNode{
-					{Id: "issueId_1_direct_dependency", Nodes: nil},
-					{Id: "issueId_2_direct_dependency", Nodes: nil},
-				},
-				},
-			},
-			expectedResult: []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"},
-		},
-	}
-
-	for _, test := range tests {
-		result := getDirectDependenciesSet(test.dependenciesTrees)
-		assert.ElementsMatch(t, test.expectedResult, result.ToSlice())
-	}
-}
-
 func TestCreateConfigFile_VerifyFileWasCreated(t *testing.T) {
 	// Arrange
 	assert.NoError(t, rtutils.DownloadAnalyzerManagerIfNeeded())
@@ -340,7 +306,7 @@ func TestCreateConfigFile_VerifyFileWasCreated(t *testing.T) {
 			assert.NoError(t, scanner.scannerDirCleanupFunc())
 		}
 	}()
-	applicabilityManager := newApplicabilityScanManager(fakeBasicXrayResults, fakeBasicDependencyGraph, scanner)
+	applicabilityManager := newApplicabilityScanManager(fakeBasicXrayResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, scanner)
 
 	currWd, err := coreutils.GetWorkingDirectory()
 	assert.NoError(t, err)

xray/audit/jas/jasmanager.go

@@ -8,7 +8,6 @@ import (
 	"github.com/jfrog/jfrog-client-go/utils/io"
 	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
-	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
 	"github.com/owenrumney/go-sarif/v2/sarif"
 	"gopkg.in/yaml.v3"
 	"os"
@@ -65,7 +64,7 @@ func (a *AdvancedSecurityScanner) Run(scannerCmd ScannerCmd) (err error) {
 	return
 }
 
-func RunScannersAndSetResults(scanResults *utils.ExtendedScanResults, dependencyTrees []*xrayUtils.GraphNode,
+func RunScannersAndSetResults(scanResults *utils.ExtendedScanResults, directDependencies []string,
 	serverDetails *config.ServerDetails, workingDirs []string, progress io.ProgressMgr) (err error) {
 	if serverDetails == nil || len(serverDetails.Url) == 0 {
 		log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.")
@@ -82,7 +81,7 @@ func RunScannersAndSetResults(scanResults *utils.ExtendedScanResults, dependency
 	if progress != nil {
 		progress.SetHeadlineMsg("Running applicability scanning")
 	}
-	scanResults.ApplicabilityScanResults, err = getApplicabilityScanResults(scanResults.XrayResults, dependencyTrees, scanResults.ScannedTechnologies, scanner)
+	scanResults.ApplicabilityScanResults, err = getApplicabilityScanResults(scanResults.XrayResults, directDependencies, scanResults.ScannedTechnologies, scanner)
 	if err != nil {
 		return
 	}

xray/audit/jas/jasmanager_test.go

@@ -73,13 +73,13 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) {
 		assert.NoError(t, os.Unsetenv(coreutils.HomeDir))
 	}()
 	scanResults := &utils.ExtendedScanResults{XrayResults: fakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Yarn}}
-	err = RunScannersAndSetResults(scanResults, fakeBasicDependencyGraph, &fakeServerDetails, nil, nil)
+	err = RunScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &fakeServerDetails, nil, nil)
 	// Expect error:
 	assert.Error(t, err)
 }
 
 func TestGetExtendedScanResults_ServerNotValid(t *testing.T) {
 	scanResults := &utils.ExtendedScanResults{XrayResults: fakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Pip}}
-	err := RunScannersAndSetResults(scanResults, fakeBasicDependencyGraph, nil, nil, nil)
+	err := RunScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, nil)
 	assert.NoError(t, err)
 }

xray/commands/audit/generic/auditmanager.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/jfrog/build-info-go/utils/pythonutils"
+	"github.com/jfrog/gofrog/datastructures"
 	rtutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"github.com/jfrog/jfrog-cli-core/v2/xray/audit"
@@ -147,7 +148,7 @@ func RunAudit(auditParams *Params) (results *Results, err error) {
 
 	// Run scanners only if the user is entitled for Advanced Security
 	if results.ExtendedScanResults.EntitledForJas {
-		results.JasError = jas.RunScannersAndSetResults(results.ExtendedScanResults, auditParams.FullDependenciesTree(), serverDetails, auditParams.workingDirs, auditParams.Progress())
+		results.JasError = jas.RunScannersAndSetResults(results.ExtendedScanResults, auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress())
 	}
 	return
 }
@@ -210,7 +211,7 @@ func runScaScanOnWorkingDir(params *Params, results *Results, workingDir, rootDi
 		if tech == coreutils.Dotnet {
 			continue
 		}
-		flattenTree, techErr := GetTechDependencyTree(params.GraphBasicParams, tech)
+		flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.GraphBasicParams, tech)
 		if techErr != nil {
 			err = errors.Join(err, fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", tech, techErr.Error()))
 			continue
@@ -231,7 +232,13 @@ func runScaScanOnWorkingDir(params *Params, results *Results, workingDir, rootDi
 			err = errors.Join(err, fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", tech, techErr.Error()))
 			continue
 		}
-		techResults = audit.BuildImpactPathsForScanResponse(techResults, params.FullDependenciesTree())
+		techResults = audit.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees)
+		if tech == coreutils.Pip {
+			params.AppendDirectDependencies(getDirectDependenciesFromTree(flattenTree))
+
+		} else {
+			params.AppendDirectDependencies(getDirectDependenciesFromTree(fullDependencyTrees))
+		}
 		results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...)
 		if !results.IsMultipleRootProject {
 			results.IsMultipleRootProject = len(flattenTree) > 1
@@ -241,43 +248,51 @@ func runScaScanOnWorkingDir(params *Params, results *Results, workingDir, rootDi
 	return
 }
 
-func GetTechDependencyTree(params *xrayutils.GraphBasicParams, tech coreutils.Technology) (flatTree []*xrayCmdUtils.GraphNode, err error) {
+// This function retrieves the dependency trees of the scanned project and extracts a set that contains only the direct dependencies.
+func getDirectDependenciesFromTree(dependencyTrees []*xrayCmdUtils.GraphNode) []string {
+	directDependencies := datastructures.MakeSet[string]()
+	for _, tree := range dependencyTrees {
+		for _, node := range tree.Nodes {
+			directDependencies.Add(node.Id)
+		}
+	}
+	return directDependencies.ToSlice()
+}
+
+func GetTechDependencyTree(params *xrayutils.GraphBasicParams, tech coreutils.Technology) (flatTree []*xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, err error) {
 	if params.Progress() != nil {
 		params.Progress().SetHeadlineMsg(fmt.Sprintf("Calculating %v dependencies", tech.ToFormal()))
 	}
 	serverDetails, err := params.ServerDetails()
 	if err != nil {
 		return
 	}
-	var dependencyTrees []*xrayCmdUtils.GraphNode
 	switch tech {
 	case coreutils.Maven, coreutils.Gradle:
-		dependencyTrees, err = getJavaDependencyTree(params, tech)
+		fullDependencyTrees, err = getJavaDependencyTree(params, tech)
 	case coreutils.Npm:
-		dependencyTrees, err = npm.BuildDependencyTree(params.Args())
+		fullDependencyTrees, err = npm.BuildDependencyTree(params.Args())
 	case coreutils.Yarn:
-		dependencyTrees, err = yarn.BuildDependencyTree()
+		fullDependencyTrees, err = yarn.BuildDependencyTree()
 	case coreutils.Go:
-		dependencyTrees, err = _go.BuildDependencyTree(serverDetails, params.DepsRepo())
+		fullDependencyTrees, err = _go.BuildDependencyTree(serverDetails, params.DepsRepo())
 	case coreutils.Pipenv, coreutils.Pip, coreutils.Poetry:
-		dependencyTrees, err = python.BuildDependencyTree(&python.AuditPython{
+		fullDependencyTrees, err = python.BuildDependencyTree(&python.AuditPython{
 			Server:              serverDetails,
 			Tool:                pythonutils.PythonTool(tech),
 			RemotePypiRepo:      params.DepsRepo(),
 			PipRequirementsFile: params.PipRequirementsFile()})
 	case coreutils.Nuget:
-		dependencyTrees, err = nuget.BuildDependencyTree()
+		fullDependencyTrees, err = nuget.BuildDependencyTree()
 	default:
 		err = errorutils.CheckErrorf("%s is currently not supported", string(tech))
 	}
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	// Save the full dependencyTree to build impact paths for vulnerable dependencies
-	params.SetFullDependenciesTree(dependencyTrees)
-
 	// Flatten the graph to speed up the ScanGraph request
-	return services.FlattenGraph(dependencyTrees)
+	flatTree, err = services.FlattenGraph(fullDependencyTrees)
+	return
 }
 
 func getJavaDependencyTree(params *xrayutils.GraphBasicParams, tech coreutils.Technology) ([]*xrayCmdUtils.GraphNode, error) {

xray/commands/audit/generic/auditmanager_test.go

@@ -0,0 +1,44 @@
+package audit
+
+import (
+	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestGetDirectDependenciesList(t *testing.T) {
+	tests := []struct {
+		dependenciesTrees []*xrayUtils.GraphNode
+		expectedResult    []string
+	}{
+		{
+			dependenciesTrees: nil,
+			expectedResult:    []string{},
+		},
+		{
+			dependenciesTrees: []*xrayUtils.GraphNode{
+				{Id: "parent_node_id", Nodes: []*xrayUtils.GraphNode{
+					{Id: "issueId_1_direct_dependency", Nodes: []*xrayUtils.GraphNode{{Id: "issueId_1_non_direct_dependency"}}},
+					{Id: "issueId_2_direct_dependency", Nodes: nil},
+				},
+				},
+			},
+			expectedResult: []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"},
+		},
+		{
+			dependenciesTrees: []*xrayUtils.GraphNode{
+				{Id: "parent_node_id", Nodes: []*xrayUtils.GraphNode{
+					{Id: "issueId_1_direct_dependency", Nodes: nil},
+					{Id: "issueId_2_direct_dependency", Nodes: nil},
+				},
+				},
+			},
+			expectedResult: []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"},
+		},
+	}
+
+	for _, test := range tests {
+		result := getDirectDependenciesFromTree(test.dependenciesTrees)
+		assert.ElementsMatch(t, test.expectedResult, result)
+	}
+}