diff --git a/artifactory/commands/golang/go.go b/artifactory/commands/golang/go.go index 93ce330e0..68ab8002f 100644 --- a/artifactory/commands/golang/go.go +++ b/artifactory/commands/golang/go.go @@ -219,7 +219,7 @@ func copyGoPackageFiles(destPath, packageName, rtTargetRepo string, authArtDetai return fmt.Errorf("couldn't find suitable package files: %s", packageFilesPath) } // Set permission recursively - return coreutils.SetPermissionsRecursively(destPath, 0700) + return coreutils.SetPermissionsRecursively(destPath, 0755) } // getPackageFilePathFromArtifactory returns a string that represents the package files cache path. diff --git a/artifactory/commands/npm/npmcommand.go b/artifactory/commands/npm/npmcommand.go index c6b97cff7..7d6061a2e 100644 --- a/artifactory/commands/npm/npmcommand.go +++ b/artifactory/commands/npm/npmcommand.go @@ -63,108 +63,108 @@ func NewNpmCiCommand() *NpmCommand { return &NpmCommand{cmdName: "ci", internalCommandName: "rt_npm_ci"} } -func (ca *NpmCommand) CommandName() string { - return ca.internalCommandName +func (nc *NpmCommand) CommandName() string { + return nc.internalCommandName } -func (ca *NpmCommand) SetConfigFilePath(configFilePath string) *NpmCommand { - ca.configFilePath = configFilePath - return ca +func (nc *NpmCommand) SetConfigFilePath(configFilePath string) *NpmCommand { + nc.configFilePath = configFilePath + return nc } -func (ca *NpmCommand) SetArgs(args []string) *NpmCommand { - ca.npmArgs = args - return ca +func (nc *NpmCommand) SetArgs(args []string) *NpmCommand { + nc.npmArgs = args + return nc } -func (ca *NpmCommand) SetRepoConfig(conf *utils.RepositoryConfig) *NpmCommand { +func (nc *NpmCommand) SetRepoConfig(conf *utils.RepositoryConfig) *NpmCommand { serverDetails, _ := conf.ServerDetails() - ca.SetRepo(conf.TargetRepo()).SetServerDetails(serverDetails) - return ca + nc.SetRepo(conf.TargetRepo()).SetServerDetails(serverDetails) + return nc } -func (ca *NpmCommand) SetServerDetails(serverDetails *config.ServerDetails) *NpmCommand { - ca.serverDetails = serverDetails - return ca +func (nc *NpmCommand) SetServerDetails(serverDetails *config.ServerDetails) *NpmCommand { + nc.serverDetails = serverDetails + return nc } -func (ca *NpmCommand) SetRepo(repo string) *NpmCommand { - ca.repo = repo - return ca +func (nc *NpmCommand) SetRepo(repo string) *NpmCommand { + nc.repo = repo + return nc } -func (ca *NpmCommand) Init() error { +func (nc *NpmCommand) Init() error { // Read config file. - log.Debug("Preparing to read the config file", ca.configFilePath) - vConfig, err := utils.ReadConfigFile(ca.configFilePath, utils.YAML) + log.Debug("Preparing to read the config file", nc.configFilePath) + vConfig, err := utils.ReadConfigFile(nc.configFilePath, utils.YAML) if err != nil { return err } // Extract resolution params. - resolverParams, err := utils.GetRepoConfigByPrefix(ca.configFilePath, utils.ProjectConfigResolverPrefix, vConfig) + resolverParams, err := utils.GetRepoConfigByPrefix(nc.configFilePath, utils.ProjectConfigResolverPrefix, vConfig) if err != nil { return err } - _, _, _, filteredNpmArgs, buildConfiguration, err := commandUtils.ExtractNpmOptionsFromArgs(ca.npmArgs) + _, _, _, filteredNpmArgs, buildConfiguration, err := commandUtils.ExtractNpmOptionsFromArgs(nc.npmArgs) if err != nil { return err } - ca.SetRepoConfig(resolverParams).SetArgs(filteredNpmArgs).SetBuildConfiguration(buildConfiguration) + nc.SetRepoConfig(resolverParams).SetArgs(filteredNpmArgs).SetBuildConfiguration(buildConfiguration) return nil } -func (ca *NpmCommand) SetBuildConfiguration(buildConfiguration *utils.BuildConfiguration) *NpmCommand { - ca.buildConfiguration = buildConfiguration - return ca +func (nc *NpmCommand) SetBuildConfiguration(buildConfiguration *utils.BuildConfiguration) *NpmCommand { + nc.buildConfiguration = buildConfiguration + return nc } -func (ca *NpmCommand) ServerDetails() (*config.ServerDetails, error) { - return ca.serverDetails, nil +func (nc *NpmCommand) ServerDetails() (*config.ServerDetails, error) { + return nc.serverDetails, nil } -func (ca *NpmCommand) RestoreNpmrcFunc() func() error { - return ca.restoreNpmrcFunc +func (nc *NpmCommand) RestoreNpmrcFunc() func() error { + return nc.restoreNpmrcFunc } -func (ca *NpmCommand) PreparePrerequisites(repo string) error { +func (nc *NpmCommand) PreparePrerequisites(repo string) error { log.Debug("Preparing prerequisites...") var err error - ca.npmVersion, ca.executablePath, err = biutils.GetNpmVersionAndExecPath(log.Logger) + nc.npmVersion, nc.executablePath, err = biutils.GetNpmVersionAndExecPath(log.Logger) if err != nil { return err } - if ca.npmVersion.Compare(minSupportedNpmVersion) > 0 { + if nc.npmVersion.Compare(minSupportedNpmVersion) > 0 { return errorutils.CheckErrorf( - "JFrog CLI npm %s command requires npm client version %s or higher. The Current version is: %s", ca.cmdName, minSupportedNpmVersion, ca.npmVersion.GetVersion()) + "JFrog CLI npm %s command requires npm client version %s or higher. The Current version is: %s", nc.cmdName, minSupportedNpmVersion, nc.npmVersion.GetVersion()) } - if err := ca.setJsonOutput(); err != nil { + if err = nc.setJsonOutput(); err != nil { return err } - ca.workingDirectory, err = coreutils.GetWorkingDirectory() + nc.workingDirectory, err = coreutils.GetWorkingDirectory() if err != nil { return err } - log.Debug("Working directory set to:", ca.workingDirectory) - if err = ca.setArtifactoryAuth(); err != nil { + log.Debug("Working directory set to:", nc.workingDirectory) + if err = nc.setArtifactoryAuth(); err != nil { return err } - ca.npmAuth, ca.registry, err = commandUtils.GetArtifactoryNpmRepoDetails(repo, &ca.authArtDetails) + nc.npmAuth, nc.registry, err = commandUtils.GetArtifactoryNpmRepoDetails(repo, &nc.authArtDetails) if err != nil { return err } - return ca.setRestoreNpmrcFunc() + return nc.setRestoreNpmrcFunc() } -func (ca *NpmCommand) setRestoreNpmrcFunc() error { - restoreNpmrcFunc, err := commandUtils.BackupFile(filepath.Join(ca.workingDirectory, npmrcFileName), npmrcBackupFileName) +func (nc *NpmCommand) setRestoreNpmrcFunc() error { + restoreNpmrcFunc, err := commandUtils.BackupFile(filepath.Join(nc.workingDirectory, npmrcFileName), npmrcBackupFileName) if err != nil { return err } - ca.restoreNpmrcFunc = func() error { + nc.restoreNpmrcFunc = func() error { if unsetEnvErr := os.Unsetenv(npmConfigAuthEnv); unsetEnvErr != nil { return unsetEnvErr } @@ -173,43 +173,43 @@ func (ca *NpmCommand) setRestoreNpmrcFunc() error { return nil } -func (ca *NpmCommand) setArtifactoryAuth() error { - authArtDetails, err := ca.serverDetails.CreateArtAuthConfig() +func (nc *NpmCommand) setArtifactoryAuth() error { + authArtDetails, err := nc.serverDetails.CreateArtAuthConfig() if err != nil { return err } if authArtDetails.GetSshAuthHeaders() != nil { return errorutils.CheckErrorf("SSH authentication is not supported in this command") } - ca.authArtDetails = authArtDetails + nc.authArtDetails = authArtDetails return nil } -func (ca *NpmCommand) setJsonOutput() error { - jsonOutput, err := npm.ConfigGet(ca.npmArgs, "json", ca.executablePath) +func (nc *NpmCommand) setJsonOutput() error { + jsonOutput, err := npm.ConfigGet(nc.npmArgs, "json", nc.executablePath) if err != nil { return err } // In case of --json=, the value of json is set to 'true', but the result from the command is not 'true' - ca.jsonOutput = jsonOutput != "false" + nc.jsonOutput = jsonOutput != "false" return nil } -func (ca *NpmCommand) processConfigLine(configLine string) (filteredLine string, err error) { +func (nc *NpmCommand) processConfigLine(configLine string) (filteredLine string, err error) { splitOption := strings.SplitN(configLine, "=", 2) key := strings.TrimSpace(splitOption[0]) validLine := len(splitOption) == 2 && isValidKey(key) if !validLine { if strings.HasPrefix(splitOption[0], "@") { // Override scoped registries (@scope = xyz) - return fmt.Sprintf("%s = %s\n", splitOption[0], ca.registry), nil + return fmt.Sprintf("%s = %s\n", splitOption[0], nc.registry), nil } return } value := strings.TrimSpace(splitOption[1]) if key == "_auth" { - return "", ca.setNpmConfigAuthEnv(value) + return "", nc.setNpmConfigAuthEnv(value) } if strings.HasPrefix(value, "[") && strings.HasSuffix(value, "]") { return addArrayConfigs(key, value), nil @@ -218,11 +218,11 @@ func (ca *NpmCommand) processConfigLine(configLine string) (filteredLine string, return fmt.Sprintf("%s\n", configLine), err } -func (ca *NpmCommand) setNpmConfigAuthEnv(value string) error { +func (nc *NpmCommand) setNpmConfigAuthEnv(value string) error { // Check if the npm version is bigger or equal to 9.3.1 - if ca.npmVersion.Compare(npmVersionForLegacyEnv) <= 0 { + if nc.npmVersion.Compare(npmVersionForLegacyEnv) <= 0 { // Get registry name without the protocol name but including the '//' - registryWithoutProtocolName := ca.registry[strings.Index(ca.registry, "://")+1:] + registryWithoutProtocolName := nc.registry[strings.Index(nc.registry, "://")+1:] // Set "npm_config_//:_auth" environment variable to allow authentication with Artifactory scopedRegistryEnv := fmt.Sprintf(npmConfigAuthEnv, registryWithoutProtocolName) return os.Setenv(scopedRegistryEnv, value) @@ -232,16 +232,16 @@ func (ca *NpmCommand) setNpmConfigAuthEnv(value string) error { return os.Setenv(npmLegacyConfigAuthEnv, value) } -func (ca *NpmCommand) prepareConfigData(data []byte) ([]byte, error) { +func (nc *NpmCommand) prepareConfigData(data []byte) ([]byte, error) { var filteredConf []string - configString := string(data) + "\n" + ca.npmAuth + configString := string(data) + "\n" + nc.npmAuth scanner := bufio.NewScanner(strings.NewReader(configString)) for scanner.Scan() { currOption := scanner.Text() if currOption == "" { continue } - filteredLine, err := ca.processConfigLine(currOption) + filteredLine, err := nc.processConfigLine(currOption) if err != nil { return nil, errorutils.CheckError(err) } @@ -253,88 +253,87 @@ func (ca *NpmCommand) prepareConfigData(data []byte) ([]byte, error) { return nil, errorutils.CheckError(err) } - filteredConf = append(filteredConf, "json = ", strconv.FormatBool(ca.jsonOutput), "\n") - filteredConf = append(filteredConf, "registry = ", ca.registry, "\n") + filteredConf = append(filteredConf, "json = ", strconv.FormatBool(nc.jsonOutput), "\n") + filteredConf = append(filteredConf, "registry = ", nc.registry, "\n") return []byte(strings.Join(filteredConf, "")), nil } -func (ca *NpmCommand) CreateTempNpmrc() error { - log.Debug("Creating project .npmrc file.") - data, err := npm.GetConfigList(ca.npmArgs, ca.executablePath) +func (nc *NpmCommand) CreateTempNpmrc() error { + data, err := npm.GetConfigList(nc.npmArgs, nc.executablePath) if err != nil { return err } - configData, err := ca.prepareConfigData(data) + configData, err := nc.prepareConfigData(data) if err != nil { return errorutils.CheckError(err) } - if err = removeNpmrcIfExists(ca.workingDirectory); err != nil { + if err = removeNpmrcIfExists(nc.workingDirectory); err != nil { return err } - - return errorutils.CheckError(os.WriteFile(filepath.Join(ca.workingDirectory, npmrcFileName), configData, 0600)) + log.Debug("Creating temporary .npmrc file.") + return errorutils.CheckError(os.WriteFile(filepath.Join(nc.workingDirectory, npmrcFileName), configData, 0755)) } -func (ca *NpmCommand) Run() (err error) { - if err = ca.PreparePrerequisites(ca.repo); err != nil { +func (nc *NpmCommand) Run() (err error) { + if err = nc.PreparePrerequisites(nc.repo); err != nil { return } defer func() { - err = errors.Join(err, ca.restoreNpmrcFunc()) + err = errors.Join(err, nc.restoreNpmrcFunc()) }() - if err = ca.CreateTempNpmrc(); err != nil { + if err = nc.CreateTempNpmrc(); err != nil { return } - if err = ca.prepareBuildInfoModule(); err != nil { + if err = nc.prepareBuildInfoModule(); err != nil { return } - err = ca.collectDependencies() + err = nc.collectDependencies() return } -func (ca *NpmCommand) prepareBuildInfoModule() error { +func (nc *NpmCommand) prepareBuildInfoModule() error { var err error - if ca.collectBuildInfo { - ca.collectBuildInfo, err = ca.buildConfiguration.IsCollectBuildInfo() + if nc.collectBuildInfo { + nc.collectBuildInfo, err = nc.buildConfiguration.IsCollectBuildInfo() if err != nil { return err } } // Build-info should not be created when installing a single package (npm install ). - if ca.collectBuildInfo && len(filterFlags(ca.npmArgs)) > 0 { + if nc.collectBuildInfo && len(filterFlags(nc.npmArgs)) > 0 { log.Info("Build-info dependencies collection is not supported for installations of single packages. Build-info creation is skipped.") - ca.collectBuildInfo = false + nc.collectBuildInfo = false } - buildName, err := ca.buildConfiguration.GetBuildName() + buildName, err := nc.buildConfiguration.GetBuildName() if err != nil { return err } - buildNumber, err := ca.buildConfiguration.GetBuildNumber() + buildNumber, err := nc.buildConfiguration.GetBuildNumber() if err != nil { return err } buildInfoService := utils.CreateBuildInfoService() - npmBuild, err := buildInfoService.GetOrCreateBuildWithProject(buildName, buildNumber, ca.buildConfiguration.GetProject()) + npmBuild, err := buildInfoService.GetOrCreateBuildWithProject(buildName, buildNumber, nc.buildConfiguration.GetProject()) if err != nil { return errorutils.CheckError(err) } - ca.buildInfoModule, err = npmBuild.AddNpmModule(ca.workingDirectory) + nc.buildInfoModule, err = npmBuild.AddNpmModule(nc.workingDirectory) if err != nil { return errorutils.CheckError(err) } - ca.buildInfoModule.SetCollectBuildInfo(ca.collectBuildInfo) - if ca.buildConfiguration.GetModule() != "" { - ca.buildInfoModule.SetName(ca.buildConfiguration.GetModule()) + nc.buildInfoModule.SetCollectBuildInfo(nc.collectBuildInfo) + if nc.buildConfiguration.GetModule() != "" { + nc.buildInfoModule.SetName(nc.buildConfiguration.GetModule()) } return nil } -func (ca *NpmCommand) collectDependencies() error { - ca.buildInfoModule.SetNpmArgs(append([]string{ca.cmdName}, ca.npmArgs...)) - return errorutils.CheckError(ca.buildInfoModule.Build()) +func (nc *NpmCommand) collectDependencies() error { + nc.buildInfoModule.SetNpmArgs(append([]string{nc.cmdName}, nc.npmArgs...)) + return errorutils.CheckError(nc.buildInfoModule.Build()) } // Gets a config with value which is an array @@ -362,7 +361,7 @@ func removeNpmrcIfExists(workingDirectory string) error { return errorutils.CheckError(err) } - log.Debug("Removing Existing .npmrc file") + log.Debug("Removing existing .npmrc file") return errorutils.CheckError(os.Remove(filepath.Join(workingDirectory, npmrcFileName))) } diff --git a/artifactory/commands/transferconfig/transferconfig.go b/artifactory/commands/transferconfig/transferconfig.go index c0e1da318..796e99c86 100644 --- a/artifactory/commands/transferconfig/transferconfig.go +++ b/artifactory/commands/transferconfig/transferconfig.go @@ -347,13 +347,11 @@ func (tcc *TransferConfigCommand) exportSourceArtifactory() (string, func() erro } // Do export - trueValue := true - falseValue := false exportParams := services.ExportParams{ ExportPath: exportPath, - IncludeMetadata: &falseValue, + IncludeMetadata: clientutils.Pointer(false), Verbose: &tcc.verbose, - ExcludeContent: &trueValue, + ExcludeContent: clientutils.Pointer(true), } cleanUp := func() error { return fileutils.RemoveTempDir(exportPath) } if err = tcc.SourceArtifactoryManager.Export(exportParams); err != nil { diff --git a/artifactory/commands/transferconfigmerge/transferconfigmerge_test.go b/artifactory/commands/transferconfigmerge/transferconfigmerge_test.go index b18a52d27..48d184c83 100644 --- a/artifactory/commands/transferconfigmerge/transferconfigmerge_test.go +++ b/artifactory/commands/transferconfigmerge/transferconfigmerge_test.go @@ -3,6 +3,7 @@ package transferconfigmerge import ( "github.com/jfrog/jfrog-client-go/access/services" artifactoryServices "github.com/jfrog/jfrog-client-go/artifactory/services" + clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/stretchr/testify/assert" "strings" "testing" @@ -62,17 +63,15 @@ func createProjects(sameKey, sameName, sameDescription, sameAdmin, sameQuotaByte if !sameDescription { targetDescription = sourceDescription + "Target" } - trueValue := true - falseValue := false if !sameAdmin { - targetAdmin.ManageMembers = &trueValue - targetAdmin.IndexResources = &trueValue + targetAdmin.ManageMembers = clientutils.Pointer(true) + targetAdmin.IndexResources = clientutils.Pointer(true) } - var sourceSoftLimit = &falseValue - var targetSoftLimit = &falseValue + var sourceSoftLimit = clientutils.Pointer(false) + var targetSoftLimit = clientutils.Pointer(false) if !sameSoftLimit { - targetSoftLimit = &trueValue + targetSoftLimit = clientutils.Pointer(true) } if !sameQuotaBytes { targetQuotaBytes += 125 @@ -83,24 +82,21 @@ func createProjects(sameKey, sameName, sameDescription, sameAdmin, sameQuotaByte } func TestCompareInterfaces(t *testing.T) { - trueValue := true - falseValue := false - first := artifactoryServices.DockerRemoteRepositoryParams{} first.RemoteRepositoryBaseParams = artifactoryServices.RemoteRepositoryBaseParams{Password: "ppppp"} first.Key = "string1" - first.BlackedOut = &trueValue - first.AssumedOfflinePeriodSecs = 1111 + first.BlackedOut = clientutils.Pointer(true) + first.AssumedOfflinePeriodSecs = clientutils.Pointer(1111) first.Environments = []string{"111", "aaa"} - first.ContentSynchronisation = &artifactoryServices.ContentSynchronisation{Enabled: &trueValue} + first.ContentSynchronisation = &artifactoryServices.ContentSynchronisation{Enabled: clientutils.Pointer(true)} second := artifactoryServices.DockerRemoteRepositoryParams{} second.RemoteRepositoryBaseParams = artifactoryServices.RemoteRepositoryBaseParams{Password: "sssss"} second.Key = "string2" - second.BlackedOut = &falseValue - second.AssumedOfflinePeriodSecs = 2222 + second.BlackedOut = clientutils.Pointer(false) + second.AssumedOfflinePeriodSecs = clientutils.Pointer(2222) second.Environments = []string{"222", "bbb"} - second.ContentSynchronisation = &artifactoryServices.ContentSynchronisation{Enabled: &falseValue} + second.ContentSynchronisation = &artifactoryServices.ContentSynchronisation{Enabled: clientutils.Pointer(false)} diff, err := compareInterfaces(first, second, filteredRepoKeys...) assert.NoError(t, err) diff --git a/artifactory/commands/transferfiles/api/types.go b/artifactory/commands/transferfiles/api/types.go index b9f0d8768..9cab62860 100644 --- a/artifactory/commands/transferfiles/api/types.go +++ b/artifactory/commands/transferfiles/api/types.go @@ -23,6 +23,10 @@ const ( Phase1 int = 0 Phase2 int = 1 Phase3 int = 2 + + maxFilesInChunk = 16 + // 1 GiB + maxBytesInChunk = 1 << 30 ) type TargetAuth struct { @@ -101,3 +105,18 @@ func (uc *UploadChunk) AppendUploadCandidateIfNeeded(file FileRepresentation, bu } uc.UploadCandidates = append(uc.UploadCandidates, file) } + +// Return true if the chunk contains at least 16 files or at least 1GiB in total +func (uc *UploadChunk) IsChunkFull() bool { + if len(uc.UploadCandidates) >= maxFilesInChunk { + return true + } + var totalSize int64 = 0 + for _, uploadCandidate := range uc.UploadCandidates { + totalSize += uploadCandidate.Size + if totalSize > maxBytesInChunk { + return true + } + } + return false +} diff --git a/artifactory/commands/transferfiles/api/types_test.go b/artifactory/commands/transferfiles/api/types_test.go new file mode 100644 index 000000000..c2ef3b207 --- /dev/null +++ b/artifactory/commands/transferfiles/api/types_test.go @@ -0,0 +1,51 @@ +package api + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAppendUploadCandidateIfNeeded(t *testing.T) { + uploadChunk := &UploadChunk{} + + // Regular file + uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: "regular-file"}, false) + assert.Len(t, uploadChunk.UploadCandidates, 1) + + // Build info + uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: "build-info.json"}, true) + assert.Len(t, uploadChunk.UploadCandidates, 2) + + // Directory in build info - should be skipped + uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{}, true) + assert.Len(t, uploadChunk.UploadCandidates, 2) +} + +var isChunkFullCases = []struct { + files []FileRepresentation + isFull bool +}{ + {[]FileRepresentation{}, false}, + {[]FileRepresentation{{Name: "slim-jim", Size: 10737418}}, false}, + {[]FileRepresentation{{Name: "fat-vinny", Size: 1073741825}}, true}, +} + +func TestIsChunkFull(t *testing.T) { + for _, testCase := range isChunkFullCases { + t.Run("", func(t *testing.T) { + uploadChunk := &UploadChunk{UploadCandidates: testCase.files} + assert.Equal(t, testCase.isFull, uploadChunk.IsChunkFull()) + }) + } +} + +func TestIsChunkFullNumberOfFiles(t *testing.T) { + uploadChunk := &UploadChunk{} + for i := 0; i < maxFilesInChunk; i++ { + assert.False(t, uploadChunk.IsChunkFull()) + uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: fmt.Sprintf("%d", i)}, false) + } + assert.True(t, uploadChunk.IsChunkFull()) +} diff --git a/artifactory/commands/transferfiles/delayedartifactshandler.go b/artifactory/commands/transferfiles/delayedartifactshandler.go index adab43f5c..31219f00e 100644 --- a/artifactory/commands/transferfiles/delayedartifactshandler.go +++ b/artifactory/commands/transferfiles/delayedartifactshandler.go @@ -152,7 +152,7 @@ func handleDelayedArtifactsFiles(filesToConsume []string, base phaseBase, delayU delayAction := func(pBase phaseBase, addedDelayFiles []string) error { // We call this method as a recursion in order to have inner order base on the comparison function list. // Remove the first delay comparison function one by one to no longer delay it until the list is empty. - if len(filesToConsume) > 0 && len(delayUploadComparisonFunctions) > 0 { + if len(addedDelayFiles) > 0 && len(delayUploadComparisonFunctions) > 0 { return handleDelayedArtifactsFiles(addedDelayFiles, pBase, delayUploadComparisonFunctions[1:]) } return nil @@ -161,6 +161,7 @@ func handleDelayedArtifactsFiles(filesToConsume []string, base phaseBase, delayU } func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToConsume []string, uploadChunkChan chan UploadedChunk, base phaseBase, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { + log.Debug(fmt.Sprintf("Starting to handle delayed artifacts files. Found %d files.", len(filesToConsume))) for _, filePath := range filesToConsume { log.Debug("Handling delayed artifacts file: '" + filePath + "'") delayedArtifactsFile, err := readDelayFile(filePath) diff --git a/artifactory/commands/transferfiles/errorshandler.go b/artifactory/commands/transferfiles/errorshandler.go index f67079039..db9a8cb3e 100644 --- a/artifactory/commands/transferfiles/errorshandler.go +++ b/artifactory/commands/transferfiles/errorshandler.go @@ -11,6 +11,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" "os" + "path" "time" ) @@ -189,7 +190,7 @@ func (mng *TransferErrorsMng) writeErrorContent(e ExtendedFileUploadStatusRespon } func (mng *TransferErrorsMng) writeSkippedErrorContent(e ExtendedFileUploadStatusResponse) error { - log.Debug(fmt.Sprintf("Writing '%s' to file %s", e.Reason, mng.errorWriterMng.skipped.filePath)) + logWritingArtifact(e, mng.errorWriterMng.skipped.filePath) mng.errorWriterMng.skipped.writer.Write(e) mng.errorWriterMng.skipped.errorCount++ // If file contains maximum number of errors - create and write to a new errors file @@ -213,7 +214,7 @@ func (mng *TransferErrorsMng) writeSkippedErrorContent(e ExtendedFileUploadStatu } func (mng *TransferErrorsMng) writeRetryableErrorContent(e ExtendedFileUploadStatusResponse) error { - log.Debug(fmt.Sprintf("Writing '%s' to file %s", e.Reason, mng.errorWriterMng.retryable.filePath)) + logWritingArtifact(e, mng.errorWriterMng.retryable.filePath) mng.errorWriterMng.retryable.writer.Write(e) mng.errorWriterMng.retryable.errorCount++ // If file contains maximum number of errors - create and write to a new errors file @@ -236,6 +237,17 @@ func (mng *TransferErrorsMng) writeRetryableErrorContent(e ExtendedFileUploadSta return nil } +func logWritingArtifact(e ExtendedFileUploadStatusResponse, errorsFilePath string) { + if log.GetLogger().GetLogLevel() != log.DEBUG { + return + } + msg := fmt.Sprintf("Writing artifact '%s' to errors file '%s'.", path.Join(e.Repo, e.Path, e.Name), errorsFilePath) + if e.Reason != "" { + msg += fmt.Sprintf(" Reason: '%s'.", e.Reason) + } + log.Debug(msg) +} + func (writerMng *errorWriter) closeWriter() error { // Close content writer and move output file to our working directory if writerMng.writer == nil { diff --git a/artifactory/commands/transferfiles/fulltransfer.go b/artifactory/commands/transferfiles/fulltransfer.go index 3229982d9..4e90d7564 100644 --- a/artifactory/commands/transferfiles/fulltransfer.go +++ b/artifactory/commands/transferfiles/fulltransfer.go @@ -239,7 +239,7 @@ func (m *fullTransferPhase) handleFoundFile(pcWrapper producerConsumerWrapper, return } curUploadChunk.AppendUploadCandidateIfNeeded(file, m.buildInfoRepo) - if len(curUploadChunk.UploadCandidates) == uploadChunkSize { + if curUploadChunk.IsChunkFull() { _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(&m.phaseBase, *curUploadChunk, uploadChunkChan, errorsChannelMng), pcWrapper.errorsQueue.AddError) if err != nil { return diff --git a/artifactory/commands/transferfiles/longpropertycheck.go b/artifactory/commands/transferfiles/longpropertycheck.go index 35b9f120f..6b067bf40 100644 --- a/artifactory/commands/transferfiles/longpropertycheck.go +++ b/artifactory/commands/transferfiles/longpropertycheck.go @@ -3,23 +3,24 @@ package transferfiles import ( "encoding/json" "fmt" + "io" + "sync" + "github.com/jfrog/gofrog/parallel" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" cmdutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/utils/progressbar" "github.com/jfrog/jfrog-client-go/artifactory" servicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" clientutils "github.com/jfrog/jfrog-client-go/utils" "golang.org/x/exp/slices" - "io" - "sync" "github.com/jfrog/jfrog-client-go/utils/log" - "github.com/jfrog/jfrog-client-go/utils/errorutils" "time" + + "github.com/jfrog/jfrog-client-go/utils/errorutils" ) const ( @@ -112,7 +113,7 @@ func (lpc *LongPropertyCheck) ExecuteCheck(args cmdutils.RunArguments) (passed b // Returns the number of long properties found func (lpc *LongPropertyCheck) longPropertiesTaskProducer(progress *progressbar.TasksProgressBar, args cmdutils.RunArguments) int { // Init - serviceManager, err := utils.CreateServiceManagerWithContext(args.Context, args.ServerDetails, false, 0, retries, retriesWaitMilliSecs) + serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails) if err != nil { return 0 } @@ -174,7 +175,7 @@ func getSearchAllPropertiesQuery(pageNumber int) string { // We keep only the files that are at the requested repos and pass them at the files channel func createSearchPropertyTask(property Property, repos []string, args cmdutils.RunArguments, filesChan chan FileWithLongProperty, progress *progressbar.TasksProgressBar) parallel.TaskFunc { return func(threadId int) (err error) { - serviceManager, err := utils.CreateServiceManagerWithContext(args.Context, args.ServerDetails, false, 0, retries, retriesWaitMilliSecs) + serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails) if err != nil { return } diff --git a/artifactory/commands/transferfiles/manager.go b/artifactory/commands/transferfiles/manager.go index 2b825f88b..e9904b592 100644 --- a/artifactory/commands/transferfiles/manager.go +++ b/artifactory/commands/transferfiles/manager.go @@ -228,6 +228,8 @@ func runProducerConsumers(pcWrapper *producerConsumerWrapper) (executionErr erro go func() { // Wait till notified that the builder has no additional tasks, and close the builder producer consumer. <-pcWrapper.chunkBuilderProducerConsumer.GetFinishedNotification() + log.Debug("Chunk builder producer consumer has completed all tasks. " + + "All files relevant to this phase were found and added to chunks that are being uploaded...") pcWrapper.chunkBuilderProducerConsumer.Done() }() @@ -261,6 +263,7 @@ func pollUploads(phaseBase *phaseBase, srcUpService *srcUserPluginService, uploa } for i := 0; ; i++ { if ShouldStop(phaseBase, nil, errorsChannelMng) { + log.Debug("Stop signal received while polling on uploads...") return } time.Sleep(waitTimeBetweenChunkStatusSeconds * time.Second) @@ -284,6 +287,7 @@ func pollUploads(phaseBase *phaseBase, srcUpService *srcUserPluginService, uploa // it will be written to the error channel if chunksLifeCycleManager.totalChunks == 0 { if shouldStopPolling(doneChan) { + log.Debug("Stopping to poll on uploads...") return } continue @@ -386,6 +390,7 @@ func handleChunksStatuses(phase *phaseBase, chunksStatus *api.UploadChunksStatus stopped := handleFilesOfCompletedChunk(chunk.Files, errorsChannelMng) // In case an error occurred while writing errors status's to the errors file - stop transferring. if stopped { + log.Debug("Stop signal received while handling chunks statuses...") return true } err = setChunkCompletedInRepoSnapshot(phase.stateManager, chunk.Files) diff --git a/artifactory/commands/transferfiles/transfer.go b/artifactory/commands/transferfiles/transfer.go index fb45a01dc..218a9fb28 100644 --- a/artifactory/commands/transferfiles/transfer.go +++ b/artifactory/commands/transferfiles/transfer.go @@ -28,7 +28,6 @@ import ( ) const ( - uploadChunkSize = 16 // Size of the channel where the transfer's go routines write the transfer errors fileWritersChannelSize = 500000 retries = 600 diff --git a/artifactory/commands/transferfiles/utils.go b/artifactory/commands/transferfiles/utils.go index 668bebc1d..902bd5421 100644 --- a/artifactory/commands/transferfiles/utils.go +++ b/artifactory/commands/transferfiles/utils.go @@ -131,7 +131,7 @@ func (m *InterruptionErr) Error() string { } func createTransferServiceManager(ctx context.Context, serverDetails *config.ServerDetails) (artifactory.ArtifactoryServicesManager, error) { - return utils.CreateServiceManagerWithContext(ctx, serverDetails, false, 0, retries, retriesWaitMilliSecs) + return utils.CreateServiceManagerWithContext(ctx, serverDetails, false, 0, retries, retriesWaitMilliSecs, time.Minute) } func createSrcRtUserPluginServiceManager(ctx context.Context, sourceRtDetails *config.ServerDetails) (*srcUserPluginService, error) { @@ -419,7 +419,7 @@ func uploadByChunks(files []api.FileRepresentation, uploadTokensChan chan Upload continue } curUploadChunk.AppendUploadCandidateIfNeeded(file, base.buildInfoRepo) - if len(curUploadChunk.UploadCandidates) == uploadChunkSize { + if curUploadChunk.IsChunkFull() { _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(&base, curUploadChunk, uploadTokensChan, errorsChannelMng), pcWrapper.errorsQueue.AddError) if err != nil { return diff --git a/artifactory/commands/utils/npmcmdutils.go b/artifactory/commands/utils/npmcmdutils.go index 883fa88d2..4ed2ea571 100644 --- a/artifactory/commands/utils/npmcmdutils.go +++ b/artifactory/commands/utils/npmcmdutils.go @@ -118,18 +118,20 @@ func ExtractNpmOptionsFromArgs(args []string) (detailedSummary, xrayScan bool, s // If there is no file at filePath, a backup file won't be created, and the restore function will delete the file at filePath. func BackupFile(filePath, backupFileName string) (restore func() error, err error) { fileInfo, err := os.Stat(filePath) - if err != nil { + if errorutils.CheckError(err) != nil { if os.IsNotExist(err) { - return createRestoreFileFunc(filePath, backupFileName), nil + restore = createRestoreFileFunc(filePath, backupFileName) + err = nil } - return nil, errorutils.CheckError(err) + return } if err = cloneFile(filePath, backupFileName, fileInfo.Mode()); err != nil { - return nil, err + return } log.Debug("The file", filePath, "was backed up successfully to", backupFileName) - return createRestoreFileFunc(filePath, backupFileName), nil + restore = createRestoreFileFunc(filePath, backupFileName) + return } func cloneFile(origFile, newName string, fileMode os.FileMode) (err error) { diff --git a/artifactory/utils/dependenciesutils.go b/artifactory/utils/dependenciesutils.go index 91fa38cd1..bb1d12850 100644 --- a/artifactory/utils/dependenciesutils.go +++ b/artifactory/utils/dependenciesutils.go @@ -215,7 +215,7 @@ func DownloadDependency(artDetails *config.ServerDetails, downloadPath, targetPa if err = errorutils.CheckResponseStatus(resp, http.StatusOK); err != nil { return err } - err = coreutils.SetPermissionsRecursively(tempDirPath, 0700) + err = coreutils.SetPermissionsRecursively(tempDirPath, 0755) if err != nil { return err } diff --git a/artifactory/utils/storageinfo.go b/artifactory/utils/storageinfo.go index be46105cd..bb512bc6b 100644 --- a/artifactory/utils/storageinfo.go +++ b/artifactory/utils/storageinfo.go @@ -35,7 +35,7 @@ type StorageInfoManager struct { } func NewStorageInfoManager(ctx context.Context, serverDetails *config.ServerDetails) (*StorageInfoManager, error) { - serviceManager, err := CreateServiceManagerWithContext(ctx, serverDetails, false, 0, serviceManagerRetriesPerRequest, serviceManagerRetriesWaitPerRequestMilliSecs) + serviceManager, err := CreateServiceManagerWithContext(ctx, serverDetails, false, 0, serviceManagerRetriesPerRequest, serviceManagerRetriesWaitPerRequestMilliSecs, time.Minute) if err != nil { return nil, err } diff --git a/artifactory/utils/utils.go b/artifactory/utils/utils.go index f20ed3c01..3730505ef 100644 --- a/artifactory/utils/utils.go +++ b/artifactory/utils/utils.go @@ -4,6 +4,14 @@ import ( "context" "encoding/json" "errors" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "time" + "github.com/jfrog/build-info-go/build" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -18,12 +26,6 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" ioUtils "github.com/jfrog/jfrog-client-go/utils/io" "github.com/jfrog/jfrog-client-go/utils/log" - "io" - "net/http" - "net/url" - "os" - "path" - "path/filepath" ) func GetProjectDir(global bool) (string, error) { @@ -84,16 +86,16 @@ func GetEncryptedPasswordFromArtifactory(artifactoryAuth auth.ServiceDetails, in } func CreateServiceManager(serverDetails *config.ServerDetails, httpRetries, httpRetryWaitMilliSecs int, isDryRun bool) (artifactory.ArtifactoryServicesManager, error) { - return CreateServiceManagerWithContext(context.Background(), serverDetails, isDryRun, 0, httpRetries, httpRetryWaitMilliSecs) + return CreateServiceManagerWithContext(context.Background(), serverDetails, isDryRun, 0, httpRetries, httpRetryWaitMilliSecs, 0) } // Create a service manager with threads. // If the value sent for httpRetries is negative, the default will be used. func CreateServiceManagerWithThreads(serverDetails *config.ServerDetails, isDryRun bool, threads, httpRetries, httpRetryWaitMilliSecs int) (artifactory.ArtifactoryServicesManager, error) { - return CreateServiceManagerWithContext(context.Background(), serverDetails, isDryRun, threads, httpRetries, httpRetryWaitMilliSecs) + return CreateServiceManagerWithContext(context.Background(), serverDetails, isDryRun, threads, httpRetries, httpRetryWaitMilliSecs, 0) } -func CreateServiceManagerWithContext(context context.Context, serverDetails *config.ServerDetails, isDryRun bool, threads, httpRetries, httpRetryWaitMilliSecs int) (artifactory.ArtifactoryServicesManager, error) { +func CreateServiceManagerWithContext(context context.Context, serverDetails *config.ServerDetails, isDryRun bool, threads, httpRetries, httpRetryWaitMilliSecs int, timeout time.Duration) (artifactory.ArtifactoryServicesManager, error) { certsPath, err := coreutils.GetJfrogCertsDir() if err != nil { return nil, err @@ -115,6 +117,9 @@ func CreateServiceManagerWithContext(context context.Context, serverDetails *con if threads > 0 { configBuilder.SetThreads(threads) } + if timeout > 0 { + configBuilder.SetOverallRequestTimeout(timeout) + } serviceConfig, err := configBuilder.Build() if err != nil { return nil, err diff --git a/buildscripts/download-jars.sh b/buildscripts/download-jars.sh index e5e6a5510..3e9f10a89 100755 --- a/buildscripts/download-jars.sh +++ b/buildscripts/download-jars.sh @@ -7,7 +7,7 @@ # https://github.com/jfrog/maven-dep-tree # Once you have updated the versions mentioned below, please execute this script from the root directory of the jfrog-cli-core to ensure the JAR files are updated. -GRADLE_DEP_TREE_VERSION="2.2.0" +GRADLE_DEP_TREE_VERSION="3.0.0" MAVEN_DEP_TREE_VERSION="1.0.0" curl -fL https://releases.jfrog.io/artifactory/oss-release-local/com/jfrog/gradle-dep-tree/${GRADLE_DEP_TREE_VERSION}/gradle-dep-tree-${GRADLE_DEP_TREE_VERSION}.jar -o xray/commands/audit/sca/java/gradle-dep-tree.jar diff --git a/general/envsetup/envsetup.go b/general/envsetup/envsetup.go index fa2fafe32..a7cc75943 100644 --- a/general/envsetup/envsetup.go +++ b/general/envsetup/envsetup.go @@ -27,10 +27,9 @@ import ( type OutputFormat string const ( - myJfrogEndPoint = "https://myjfrog-api.jfrog.com/api/v1/activation/cloud/cli/getStatus/" - syncSleepInterval = 5 * time.Second // 5 seconds - maxWaitMinutes = 30 * time.Minute // 30 minutes - nonExpiredTokenValue = 0 // Access Tokens with 0 expiration value are actually generated by Access with 1 year expiration. + myJfrogEndPoint = "https://myjfrog-api.jfrog.com/api/v1/activation/cloud/cli/getStatus/" + syncSleepInterval = 5 * time.Second // 5 seconds + maxWaitMinutes = 30 * time.Minute // 30 minutes // OutputFormat values Human OutputFormat = "human" @@ -47,8 +46,6 @@ const ( " jf docker scan :" ) -var trueValue = true - type EnvSetupCommand struct { registrationURL string // In case encodedConnectionDetails were provided - we have a registered user that was invited to the platform. @@ -267,8 +264,9 @@ func GenerateNewLongTermRefreshableAccessToken(server *config.ServerDetails) (er func createLongExpirationRefreshableTokenParams() *services.CreateTokenParams { params := services.CreateTokenParams{} - params.ExpiresIn = nonExpiredTokenValue - params.Refreshable = &trueValue + // Using the platform's default expiration (1 year by default). + params.ExpiresIn = nil + params.Refreshable = clientUtils.Pointer(true) params.Audience = "*@*" return ¶ms } diff --git a/general/invite/invite.go b/general/invite/invite.go index 4b8316d9f..84bdda983 100644 --- a/general/invite/invite.go +++ b/general/invite/invite.go @@ -6,6 +6,7 @@ import ( "github.com/jfrog/jfrog-client-go/access" accessservices "github.com/jfrog/jfrog-client-go/access/services" "github.com/jfrog/jfrog-client-go/artifactory/services" + clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" "strings" @@ -73,20 +74,18 @@ func (ic *InviteCommand) Run() (err error) { } func (ic *InviteCommand) createNewInvitedUser() *services.User { - var trueValue = true - var falseValue = false userDetails := services.User{} // Parameters "name" and "email" should both be with the email value for internal reasons in access. userDetails.Email = ic.invitedEmail userDetails.Name = ic.invitedEmail // Random valid password - information won't be used in access. userDetails.Password = "Password1!" - userDetails.Admin = &trueValue - userDetails.ShouldInvite = &trueValue + userDetails.Admin = clientutils.Pointer(true) + userDetails.ShouldInvite = clientutils.Pointer(true) userDetails.Source = accessservices.InviteCliSourceName - userDetails.ProfileUpdatable = &trueValue - userDetails.DisableUIAccess = &falseValue - userDetails.InternalPasswordDisabled = &falseValue + userDetails.ProfileUpdatable = clientutils.Pointer(true) + userDetails.DisableUIAccess = clientutils.Pointer(false) + userDetails.InternalPasswordDisabled = clientutils.Pointer(false) return &userDetails } diff --git a/general/token/accesstokencreate.go b/general/token/accesstokencreate.go new file mode 100644 index 000000000..e1a7c4226 --- /dev/null +++ b/general/token/accesstokencreate.go @@ -0,0 +1,150 @@ +package token + +import ( + "encoding/json" + rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-client-go/access/services" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "strings" +) + +const ( + AdminScope = "applied-permissions/admin" + GroupsScopePrefix = "applied-permissions/groups:" +) + +type AccessTokenCreateCommand struct { + serverDetails *config.ServerDetails + username string + projectKey string + + scope string + groups string + grantAdmin bool + + expiry *uint + refreshable bool + description string + + audience string + includeReferenceToken bool + + response *auth.CreateTokenResponseData +} + +func NewAccessTokenCreateCommand() *AccessTokenCreateCommand { + return &AccessTokenCreateCommand{response: new(auth.CreateTokenResponseData)} +} + +func (atc *AccessTokenCreateCommand) SetServerDetails(serverDetails *config.ServerDetails) *AccessTokenCreateCommand { + atc.serverDetails = serverDetails + return atc +} + +func (atc *AccessTokenCreateCommand) SetUsername(username string) *AccessTokenCreateCommand { + atc.username = username + return atc +} + +func (atc *AccessTokenCreateCommand) SetProjectKey(projectKey string) *AccessTokenCreateCommand { + atc.projectKey = projectKey + return atc +} + +func (atc *AccessTokenCreateCommand) SetGroups(groups string) *AccessTokenCreateCommand { + atc.groups = groups + return atc +} + +func (atc *AccessTokenCreateCommand) SetScope(scope string) *AccessTokenCreateCommand { + atc.scope = scope + return atc +} + +func (atc *AccessTokenCreateCommand) SetGrantAdmin(grantAdmin bool) *AccessTokenCreateCommand { + atc.grantAdmin = grantAdmin + return atc +} + +func (atc *AccessTokenCreateCommand) SetExpiry(expiry *uint) *AccessTokenCreateCommand { + atc.expiry = expiry + return atc +} + +func (atc *AccessTokenCreateCommand) SetRefreshable(refreshable bool) *AccessTokenCreateCommand { + atc.refreshable = refreshable + return atc +} + +func (atc *AccessTokenCreateCommand) SetDescription(description string) *AccessTokenCreateCommand { + atc.description = description + return atc +} + +func (atc *AccessTokenCreateCommand) SetAudience(audience string) *AccessTokenCreateCommand { + atc.audience = audience + return atc +} + +func (atc *AccessTokenCreateCommand) SetIncludeReferenceToken(includeReferenceToken bool) *AccessTokenCreateCommand { + atc.includeReferenceToken = includeReferenceToken + return atc +} + +func (atc *AccessTokenCreateCommand) Response() ([]byte, error) { + content, err := json.Marshal(*atc.response) + return content, errorutils.CheckError(err) +} + +func (atc *AccessTokenCreateCommand) ServerDetails() (*config.ServerDetails, error) { + return atc.serverDetails, nil +} + +func (atc *AccessTokenCreateCommand) CommandName() string { + return "jf_access_token_create" +} + +func (atc *AccessTokenCreateCommand) Run() error { + servicesManager, err := rtUtils.CreateAccessServiceManager(atc.serverDetails, false) + if err != nil { + return err + } + + *atc.response, err = servicesManager.CreateAccessToken(atc.getTokenParams()) + return err +} + +func (atc *AccessTokenCreateCommand) getTokenParams() services.CreateTokenParams { + tokenParams := services.CreateTokenParams{} + + tokenParams.Username = strings.ToLower(atc.username) + tokenParams.ProjectKey = atc.projectKey + tokenParams.Scope = atc.getScope() + tokenParams.ExpiresIn = atc.expiry + tokenParams.Refreshable = &atc.refreshable + tokenParams.Description = atc.description + tokenParams.Audience = atc.audience + tokenParams.IncludeReferenceToken = &atc.includeReferenceToken + return tokenParams +} + +// If an explicit scope was provided, apply it. +// Otherwise, if admin or groups scopes were requested, construct scope from them (space separated). +// If no scopes were requested, leave scope empty to provide the default user scope. +func (atc *AccessTokenCreateCommand) getScope() string { + if atc.scope != "" { + return atc.scope + } + + var scopes []string + if atc.groups != "" { + scopes = append(scopes, GroupsScopePrefix+atc.groups) + } + + if atc.grantAdmin { + scopes = append(scopes, AdminScope) + } + return strings.Join(scopes, " ") +} diff --git a/go.mod b/go.mod index f4b687efc..20607002b 100644 --- a/go.mod +++ b/go.mod @@ -12,12 +12,13 @@ require ( github.com/google/uuid v1.3.1 github.com/gookit/color v1.5.4 github.com/jedib0t/go-pretty/v6 v6.4.7 - github.com/jfrog/build-info-go v1.9.10 + github.com/jfrog/build-info-go v1.9.12 github.com/jfrog/gofrog v1.3.0 - github.com/jfrog/jfrog-client-go v1.32.3 + github.com/jfrog/jfrog-apps-config v1.0.1 + github.com/jfrog/jfrog-client-go v1.34.0 github.com/magiconair/properties v1.8.7 github.com/manifoldco/promptui v0.9.0 - github.com/owenrumney/go-sarif/v2 v2.2.0 + github.com/owenrumney/go-sarif/v2 v2.2.2 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 @@ -36,7 +37,7 @@ require ( github.com/BurntSushi/toml v1.3.2 // indirect github.com/CycloneDX/cyclonedx-go v0.7.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/acomagu/bufpipe v1.0.4 // indirect @@ -93,3 +94,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) + +// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9 + +// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231003094520-3a09931ceaa8 diff --git a/go.sum b/go.sum index 737b839c2..fc5614fdc 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,8 @@ github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7B github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c h1:kMFnB0vCcX7IL/m9Y5LO+KQYv+t1CQOiFe6+SV2J7bE= +github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -195,12 +195,14 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedib0t/go-pretty/v6 v6.4.7 h1:lwiTJr1DEkAgzljsUsORmWsVn5MQjt1BPJdPCtJ6KXE= github.com/jedib0t/go-pretty/v6 v6.4.7/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= -github.com/jfrog/build-info-go v1.9.10 h1:uXnDLVxpqxoAMpXcki00QaBB+M2BoGMMpHODPkmmYOY= -github.com/jfrog/build-info-go v1.9.10/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg= +github.com/jfrog/build-info-go v1.9.12 h1:EixG8LrRy1GN8oaYbmOcbqKDzsch5e7nUhk4Pwb+BAQ= +github.com/jfrog/build-info-go v1.9.12/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= -github.com/jfrog/jfrog-client-go v1.32.3 h1:B2M8Gu8EMrokbHWPPDgN1b7YRWwf0oe746epvQASK6c= -github.com/jfrog/jfrog-client-go v1.32.3/go.mod h1:UewnwkIf/77HzBgwCPzOHZCK6V/Nw5/JwdzN/tRb4aU= +github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= +github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= +github.com/jfrog/jfrog-client-go v1.34.0 h1:G+H73T3blf8DQkNZJFIfqlAnF4g+qOFL+IRlbPwD6GY= +github.com/jfrog/jfrog-client-go v1.34.0/go.mod h1:o1YDnyk8j7CWly23dbQZI9j7mSYcBxrPSfBtkul4N+0= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -251,8 +253,8 @@ github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7 github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= -github.com/owenrumney/go-sarif/v2 v2.2.0 h1:1DmZaijK0HBZCR1fgcDSGa7VzYkU9NDmbZ7qC2QfUjE= -github.com/owenrumney/go-sarif/v2 v2.2.0/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= +github.com/owenrumney/go-sarif/v2 v2.2.2 h1:x2acaiiAW9hu+78wbEYBRGLk5nRtHmkv7HeUsKvblwc= +github.com/owenrumney/go-sarif/v2 v2.2.2/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= @@ -319,7 +321,7 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 58ea37a44..3880da6ad 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -44,8 +44,8 @@ type TechData struct { ciSetupSupport bool // Whether Contextual Analysis supported in this technology. applicabilityScannable bool - // The file that handles the project's dependencies. - packageDescriptor string + // The files that handle the project's dependencies. + packageDescriptors []string // Formal name of the technology formal string // The executable name of the technology @@ -60,21 +60,21 @@ var technologiesData = map[Technology]TechData{ Maven: { indicators: []string{"pom.xml"}, ciSetupSupport: true, - packageDescriptor: "pom.xml", + packageDescriptors: []string{"pom.xml"}, execCommand: "mvn", applicabilityScannable: true, }, Gradle: { indicators: []string{".gradle", ".gradle.kts"}, ciSetupSupport: true, - packageDescriptor: "build.gradle, build.gradle.kts", + packageDescriptors: []string{"build.gradle", "build.gradle.kts"}, applicabilityScannable: true, }, Npm: { indicators: []string{"package.json", "package-lock.json", "npm-shrinkwrap.json"}, exclude: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, ciSetupSupport: true, - packageDescriptor: "package.json", + packageDescriptors: []string{"package.json"}, formal: string(Npm), packageVersionOperator: "@", packageInstallationCommand: "install", @@ -82,26 +82,27 @@ var technologiesData = map[Technology]TechData{ }, Yarn: { indicators: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, - packageDescriptor: "package.json", + packageDescriptors: []string{"package.json"}, packageVersionOperator: "@", applicabilityScannable: true, }, Go: { indicators: []string{"go.mod"}, - packageDescriptor: "go.mod", + packageDescriptors: []string{"go.mod"}, packageVersionOperator: "@v", packageInstallationCommand: "get", }, Pip: { packageType: Pypi, indicators: []string{"setup.py", "requirements.txt"}, + packageDescriptors: []string{"setup.py", "requirements.txt"}, exclude: []string{"Pipfile", "Pipfile.lock", "pyproject.toml", "poetry.lock"}, applicabilityScannable: true, }, Pipenv: { packageType: Pypi, indicators: []string{"Pipfile", "Pipfile.lock"}, - packageDescriptor: "Pipfile", + packageDescriptors: []string{"Pipfile"}, packageVersionOperator: "==", packageInstallationCommand: "install", applicabilityScannable: true, @@ -153,11 +154,8 @@ func (tech Technology) GetPackageType() string { return technologiesData[tech].packageType } -func (tech Technology) GetPackageDescriptor() string { - if technologiesData[tech].packageDescriptor == "" { - return tech.ToFormal() + " Package Descriptor" - } - return technologiesData[tech].packageDescriptor +func (tech Technology) GetPackageDescriptor() []string { + return technologiesData[tech].packageDescriptors } func (tech Technology) IsCiSetup() bool { diff --git a/utils/usage/usage.go b/utils/usage/usage.go index 30f40f18c..9b493637b 100644 --- a/utils/usage/usage.go +++ b/utils/usage/usage.go @@ -2,7 +2,6 @@ package usage import ( "fmt" - "golang.org/x/sync/errgroup" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" @@ -12,7 +11,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" ecosysusage "github.com/jfrog/jfrog-client-go/utils/usage" - xrayusage "github.com/jfrog/jfrog-client-go/xray/usage" + "golang.org/x/sync/errgroup" ) const ( @@ -47,7 +46,6 @@ func NewUsageReporter(productId string, serverDetails *config.ServerDetails) *Us serverDetails: serverDetails, reportWaitGroup: new(errgroup.Group), sendToEcosystem: true, - sendToXray: true, sendToArtifactory: true, } } @@ -110,12 +108,15 @@ func (ur *UsageReporter) WaitForResponses() (err error) { func (ur *UsageReporter) reportToEcosystem(features ...ReportFeature) (err error) { if ur.serverDetails.Url == "" { - err = errorutils.CheckErrorf("platform Url is not set") + err = errorutils.CheckErrorf("platform URL is not set") return } reports, err := ur.convertAttributesToEcosystemReports(features...) - if len(reports) == 0 || err != nil { - err = errorutils.CheckErrorf("Nothing to send.") + if err != nil { + return + } + if len(reports) == 0 { + err = errorutils.CheckErrorf("nothing to send") return } return ecosysusage.SendEcosystemUsageReports(reports...) @@ -123,7 +124,7 @@ func (ur *UsageReporter) reportToEcosystem(features ...ReportFeature) (err error func (ur *UsageReporter) reportToArtifactory(features ...ReportFeature) (err error) { if ur.serverDetails.ArtifactoryUrl == "" { - err = errorutils.CheckErrorf("Artifactory Url is not set..") + err = errorutils.CheckErrorf("Artifactory URL is not set") return } serviceManager, err := utils.CreateServiceManager(ur.serverDetails, -1, 0, false) @@ -132,7 +133,7 @@ func (ur *UsageReporter) reportToArtifactory(features ...ReportFeature) (err err } converted := ur.convertAttributesToArtifactoryFeatures(features...) if len(converted) == 0 { - err = errorutils.CheckErrorf("Nothing to send.") + err = errorutils.CheckErrorf("nothing to send") return } return usage.ReportUsageToArtifactory(ur.ProductId, serviceManager, converted...) @@ -163,29 +164,6 @@ func (ur *UsageReporter) convertAttributesToArtifactoryFeatures(reportFeatures . return } -func (ur *UsageReporter) convertAttributesToXrayEvents(reportFeatures ...ReportFeature) (events []xrayusage.ReportXrayEventData) { - for _, feature := range reportFeatures { - convertedAttributes := []xrayusage.ReportUsageAttribute{} - for _, attribute := range feature.Attributes { - convertedAttributes = append(convertedAttributes, xrayusage.ReportUsageAttribute{ - AttributeName: attribute.AttributeName, - AttributeValue: attribute.AttributeValue, - }) - } - if feature.ClientId != "" { - // Add clientId as attribute - convertedAttributes = append(convertedAttributes, xrayusage.ReportUsageAttribute{ - AttributeName: clientIdAttributeName, - AttributeValue: feature.ClientId, - }) - } - events = append(events, xrayusage.CreateUsageEvent( - ur.ProductId, feature.FeatureId, convertedAttributes..., - )) - } - return -} - func (ur *UsageReporter) convertAttributesToEcosystemReports(reportFeatures ...ReportFeature) (reports []ecosysusage.ReportEcosystemUsageData, err error) { accountId := ur.serverDetails.Url clientToFeaturesMap := map[string][]string{} diff --git a/utils/usage/usage_test.go b/utils/usage/usage_test.go index 657f19ba1..c3b2b08a1 100644 --- a/utils/usage/usage_test.go +++ b/utils/usage/usage_test.go @@ -10,7 +10,6 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-client-go/artifactory/usage" ecosysusage "github.com/jfrog/jfrog-client-go/utils/usage" - xrayusage "github.com/jfrog/jfrog-client-go/xray/usage" "github.com/stretchr/testify/assert" ) @@ -40,28 +39,6 @@ var ( {FeatureId: "featureId", ClientId: "clientId2"}, {FeatureId: "featureId"}, } - xrayEvents = []xrayusage.ReportXrayEventData{ - { - ProductId: productName, - EventId: "server_" + productName + "_featureId2", - Attributes: map[string]string{ - "clientId": "clientId", - "attribute": "value", - }, - Origin: "API_CLI", - }, - { - ProductId: productName, - EventId: "server_" + productName + "_featureId", - Attributes: map[string]string{"clientId": "clientId2"}, - Origin: "API_CLI", - }, - { - ProductId: productName, - EventId: "server_" + productName + "_featureId", - Origin: "API_CLI", - }, - } ecosystemData = []ecosysusage.ReportEcosystemUsageData{ { ProductId: productName, @@ -90,13 +67,6 @@ func TestConvertToArtifactoryUsage(t *testing.T) { } } -func TestConvertToXrayUsage(t *testing.T) { - reporter := NewUsageReporter(productName, &config.ServerDetails{XrayUrl: serverUrl + "/"}) - for i := 0; i < len(features); i++ { - assert.Equal(t, xrayEvents[i], reporter.convertAttributesToXrayEvents(features[i])[0]) - } -} - func TestConvertToEcosystemUsage(t *testing.T) { reporter := NewUsageReporter(productName, &config.ServerDetails{Url: serverUrl}) for i := 0; i < len(features); i++ { @@ -173,7 +143,7 @@ func TestReportEcosystemUsageError(t *testing.T) { } func create404UsageHandler(t *testing.T) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotFound) } } diff --git a/xray/commands/audit/audit.go b/xray/commands/audit/audit.go index 2303ef0a2..1347e5357 100644 --- a/xray/commands/audit/audit.go +++ b/xray/commands/audit/audit.go @@ -109,15 +109,15 @@ func (auditCmd *AuditCommand) Run() (err error) { // Print Scan results on all cases except if errors accrued on SCA scan and no security/license issues found. printScanResults := !(auditResults.ScaError != nil && xrayutils.IsEmptyScanResponse(auditResults.ExtendedScanResults.XrayResults)) if printScanResults { - err = xrayutils.PrintScanResults(auditResults.ExtendedScanResults, - nil, - auditCmd.OutputFormat(), - auditCmd.IncludeVulnerabilities, - auditCmd.IncludeLicenses, - auditResults.IsMultipleRootProject, - auditCmd.PrintExtendedTable, false, messages, - ) - if err != nil { + if err = xrayutils.NewResultsWriter(auditResults.ExtendedScanResults). + SetIsMultipleRootProject(auditResults.IsMultipleRootProject). + SetIncludeVulnerabilities(auditCmd.IncludeVulnerabilities). + SetIncludeLicenses(auditCmd.IncludeLicenses). + SetOutputFormat(auditCmd.OutputFormat()). + SetPrintExtendedTable(auditCmd.PrintExtendedTable). + SetExtraMessages(messages). + SetScanType(services.Dependency). + PrintScanResults(); err != nil { return } } diff --git a/xray/commands/audit/jas/applicability/applicabilitymanager.go b/xray/commands/audit/jas/applicability/applicabilitymanager.go index b038e4371..202e3d9ff 100644 --- a/xray/commands/audit/jas/applicability/applicabilitymanager.go +++ b/xray/commands/audit/jas/applicability/applicabilitymanager.go @@ -3,6 +3,7 @@ package applicability import ( "path/filepath" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/gofrog/datastructures" @@ -16,8 +17,9 @@ import ( ) const ( - applicabilityScanType = "analyze-applicability" - applicabilityScanCommand = "ca" + applicabilityScanType = "analyze-applicability" + applicabilityScanCommand = "ca" + applicabilityDocsUrlSuffix = "contextual-analysis" ) type ApplicabilityScanManager struct { @@ -100,19 +102,22 @@ func isDirectComponents(components []string, directDependencies []string) bool { return false } -func (asm *ApplicabilityScanManager) Run(wd string) (err error) { - if len(asm.scanner.WorkingDirs) > 1 { - log.Info("Running applicability scanning in the", wd, "directory...") +func (asm *ApplicabilityScanManager) Run(module jfrogappsconfig.Module) (err error) { + if jas.ShouldSkipScanner(module, utils.Applicability) { + return + } + if len(asm.scanner.JFrogAppsConfig.Modules) > 1 { + log.Info("Running applicability scanning in the", module.SourceRoot, "directory...") } else { log.Info("Running applicability scanning...") } - if err = asm.createConfigFile(wd); err != nil { + if err = asm.createConfigFile(module); err != nil { return } if err = asm.runAnalyzerManager(); err != nil { return } - workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.scanner.ResultsFileName, wd) + workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.scanner.ResultsFileName, module.SourceRoot, applicabilityDocsUrlSuffix) if err != nil { return } @@ -141,25 +146,29 @@ type scanConfiguration struct { SkippedDirs []string `yaml:"skipped-folders"` } -func (asm *ApplicabilityScanManager) createConfigFile(workingDir string) error { - skipDirs := jas.SkippedDirs +func (asm *ApplicabilityScanManager) createConfigFile(module jfrogappsconfig.Module) error { + roots, err := jas.GetSourceRoots(module, nil) + if err != nil { + return err + } + excludePatterns := jas.GetExcludePatterns(module, nil) if asm.thirdPartyScan { log.Info("Including node modules folder in applicability scan") - skipDirs = removeElementFromSlice(skipDirs, jas.NodeModulesPattern) + excludePatterns = removeElementFromSlice(excludePatterns, jas.NodeModulesPattern) } configFileContent := applicabilityScanConfig{ Scans: []scanConfiguration{ { - Roots: []string{workingDir}, + Roots: roots, Output: asm.scanner.ResultsFileName, Type: applicabilityScanType, GrepDisable: false, CveWhitelist: asm.directDependenciesCves, - SkippedDirs: skipDirs, + SkippedDirs: excludePatterns, }, }, } - return jas.CreateScannersConfigFile(asm.scanner.ConfigFileName, configFileContent) + return jas.CreateScannersConfigFile(asm.scanner.ConfigFileName, configFileContent, utils.Applicability) } // Runs the analyzerManager app and returns a boolean to indicate whether the user is entitled for diff --git a/xray/commands/audit/jas/applicability/applicabilitymanager_test.go b/xray/commands/audit/jas/applicability/applicabilitymanager_test.go index b05d69130..554909181 100644 --- a/xray/commands/audit/jas/applicability/applicabilitymanager_test.go +++ b/xray/commands/audit/jas/applicability/applicabilitymanager_test.go @@ -1,13 +1,15 @@ package applicability import ( + "os" + "path/filepath" + "testing" + + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/jfrog-client-go/xray/services" "github.com/stretchr/testify/assert" - "os" - "path/filepath" - "testing" ) var mockDirectDependencies = []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"} @@ -36,7 +38,7 @@ func TestNewApplicabilityScanManager_DependencyTreeDoesntExist(t *testing.T) { // Assert if assert.NotNil(t, applicabilityManager) { assert.NotNil(t, applicabilityManager.scanner.ScannerDirCleanupFunc) - assert.Len(t, applicabilityManager.scanner.WorkingDirs, 1) + assert.Len(t, applicabilityManager.scanner.JFrogAppsConfig.Modules, 1) assert.NotEmpty(t, applicabilityManager.scanner.ConfigFileName) assert.NotEmpty(t, applicabilityManager.scanner.ResultsFileName) assert.Empty(t, applicabilityManager.directDependenciesCves) @@ -255,7 +257,7 @@ func TestCreateConfigFile_VerifyFileWasCreated(t *testing.T) { currWd, err := coreutils.GetWorkingDirectory() assert.NoError(t, err) - err = applicabilityManager.createConfigFile(currWd) + err = applicabilityManager.createConfigFile(jfrogappsconfig.Module{SourceRoot: currWd}) assert.NoError(t, err) defer func() { @@ -280,7 +282,7 @@ func TestParseResults_EmptyResults_AllCvesShouldGetUnknown(t *testing.T) { // Act var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) if assert.NoError(t, err) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) @@ -297,7 +299,7 @@ func TestParseResults_ApplicableCveExist(t *testing.T) { // Act var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) @@ -314,7 +316,7 @@ func TestParseResults_AllCvesNotApplicable(t *testing.T) { // Act var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) diff --git a/xray/commands/audit/jas/common.go b/xray/commands/audit/jas/common.go index 2dfada8dc..be165aa32 100644 --- a/xray/commands/audit/jas/common.go +++ b/xray/commands/audit/jas/common.go @@ -2,20 +2,25 @@ package jas import ( "errors" + "fmt" "os" "path/filepath" "strings" "testing" + "unicode" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" rtutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray/services" "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" + "golang.org/x/exp/slices" "gopkg.in/yaml.v3" ) @@ -24,7 +29,7 @@ const ( ) var ( - SkippedDirs = []string{"**/*test*/**", "**/*venv*/**", NodeModulesPattern, "**/*target*/**"} + DefaultExcludePatterns = []string{"**/*test*/**", "**/*venv*/**", NodeModulesPattern, "**/*target*/**"} mapSeverityToScore = map[string]string{ "": "0.0", @@ -41,7 +46,7 @@ type JasScanner struct { ResultsFileName string AnalyzerManager utils.AnalyzerManager ServerDetails *config.ServerDetails - WorkingDirs []string + JFrogAppsConfig *jfrogappsconfig.JFrogAppsConfig ScannerDirCleanupFunc func() error } @@ -60,22 +65,42 @@ func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails, mu scanner.ServerDetails = serverDetails scanner.ConfigFileName = filepath.Join(tempDir, "config.yaml") scanner.ResultsFileName = filepath.Join(tempDir, "results.sarif") - scanner.WorkingDirs, err = coreutils.GetFullPathsWorkingDirs(workingDirs) + scanner.JFrogAppsConfig, err = createJFrogAppsConfig(workingDirs) scanner.AnalyzerManager.MultiScanId = multiScanId return } +func createJFrogAppsConfig(workingDirs []string) (*jfrogappsconfig.JFrogAppsConfig, error) { + if jfrogAppsConfig, err := jfrogappsconfig.LoadConfigIfExist(); err != nil { + return nil, errorutils.CheckError(err) + } else if jfrogAppsConfig != nil { + // jfrog-apps-config.yml exist in the workspace + return jfrogAppsConfig, nil + } + + // jfrog-apps-config.yml does not exist in the workspace + fullPathsWorkingDirs, err := coreutils.GetFullPathsWorkingDirs(workingDirs) + if err != nil { + return nil, err + } + jfrogAppsConfig := new(jfrogappsconfig.JFrogAppsConfig) + for _, workingDir := range fullPathsWorkingDirs { + jfrogAppsConfig.Modules = append(jfrogAppsConfig.Modules, jfrogappsconfig.Module{SourceRoot: workingDir}) + } + return jfrogAppsConfig, nil +} + type ScannerCmd interface { - Run(wd string) (err error) + Run(module jfrogappsconfig.Module) (err error) } func (a *JasScanner) Run(scannerCmd ScannerCmd) (err error) { - for _, workingDir := range a.WorkingDirs { + for _, module := range a.JFrogAppsConfig.Modules { func() { defer func() { err = errors.Join(err, deleteJasProcessFiles(a.ConfigFileName, a.ResultsFileName)) }() - if err = scannerCmd.Run(workingDir); err != nil { + if err = scannerCmd.Run(module); err != nil { return } }() @@ -103,7 +128,7 @@ func deleteJasProcessFiles(configFile string, resultFile string) error { return errorutils.CheckError(err) } -func ReadJasScanRunsFromFile(fileName, wd string) (sarifRuns []*sarif.Run, err error) { +func ReadJasScanRunsFromFile(fileName, wd, informationUrlSuffix string) (sarifRuns []*sarif.Run, err error) { if sarifRuns, err = utils.ReadScanRunsFromFile(fileName); err != nil { return } @@ -113,12 +138,31 @@ func ReadJasScanRunsFromFile(fileName, wd string) (sarifRuns []*sarif.Run, err e // Also used to calculate relative paths if needed with it sarifRun.Invocations[0].WorkingDirectory.WithUri(wd) // Process runs values + fillMissingRequiredDriverInformation(utils.BaseDocumentationURL+informationUrlSuffix, utils.GetAnalyzerManagerVersion(), sarifRun) sarifRun.Results = excludeSuppressResults(sarifRun.Results) addScoreToRunRules(sarifRun) } return } +func fillMissingRequiredDriverInformation(defaultJasInformationUri, defaultVersion string, run *sarif.Run) { + driver := run.Tool.Driver + if driver.InformationURI == nil { + driver.InformationURI = &defaultJasInformationUri + } + if driver.Version == nil || !isValidVersion(*driver.Version) { + driver.Version = &defaultVersion + } +} + +func isValidVersion(version string) bool { + if len(version) == 0 { + return false + } + firstChar := rune(version[0]) + return unicode.IsDigit(firstChar) +} + func excludeSuppressResults(sarifResults []*sarif.Result) []*sarif.Result { results := []*sarif.Result{} for _, sarifResult := range sarifResults { @@ -153,11 +197,12 @@ func convertToScore(severity string) string { return "" } -func CreateScannersConfigFile(fileName string, fileContent interface{}) error { +func CreateScannersConfigFile(fileName string, fileContent interface{}, scanType utils.JasScanType) error { yamlData, err := yaml.Marshal(&fileContent) if errorutils.CheckError(err) != nil { return err } + log.Debug(scanType.String() + " scanner input YAML:\n" + string(yamlData)) err = os.WriteFile(fileName, yamlData, 0644) return errorutils.CheckError(err) } @@ -196,3 +241,38 @@ func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) { func GetTestDataPath() string { return filepath.Join("..", "..", "..", "testdata") } + +func ShouldSkipScanner(module jfrogappsconfig.Module, scanType utils.JasScanType) bool { + lowerScanType := strings.ToLower(string(scanType)) + if slices.Contains(module.ExcludeScanners, lowerScanType) { + log.Info(fmt.Sprintf("Skipping %s scanning", scanType)) + return true + } + return false +} + +func GetSourceRoots(module jfrogappsconfig.Module, scanner *jfrogappsconfig.Scanner) ([]string, error) { + root, err := filepath.Abs(module.SourceRoot) + if err != nil { + return []string{}, errorutils.CheckError(err) + } + if scanner == nil || len(scanner.WorkingDirs) == 0 { + return []string{root}, errorutils.CheckError(err) + } + var roots []string + for _, workingDir := range scanner.WorkingDirs { + roots = append(roots, filepath.Join(root, workingDir)) + } + return roots, nil +} + +func GetExcludePatterns(module jfrogappsconfig.Module, scanner *jfrogappsconfig.Scanner) []string { + excludePatterns := module.ExcludePatterns + if scanner != nil { + excludePatterns = append(excludePatterns, scanner.ExcludePatterns...) + } + if len(excludePatterns) == 0 { + return DefaultExcludePatterns + } + return excludePatterns +} diff --git a/xray/commands/audit/jas/common_test.go b/xray/commands/audit/jas/common_test.go new file mode 100644 index 000000000..5466039d8 --- /dev/null +++ b/xray/commands/audit/jas/common_test.go @@ -0,0 +1,90 @@ +package jas + +import ( + "testing" + + "github.com/jfrog/jfrog-cli-core/v2/xray/utils" + "github.com/owenrumney/go-sarif/v2/sarif" + "github.com/stretchr/testify/assert" +) + +func TestExcludeSuppressResults(t *testing.T) { + tests := []struct { + name string + sarifResults []*sarif.Result + expectedOutput []*sarif.Result + }{ + { + sarifResults: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet1", "ruleId1", "level1"), + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2"), + }, + expectedOutput: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet1", "ruleId1", "level1"), + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2"), + }, + }, + { + sarifResults: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet1", "ruleId1", "level1").WithSuppression([]*sarif.Suppression{sarif.NewSuppression("")}), + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2"), + }, + expectedOutput: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2"), + }, + }, + { + sarifResults: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet1", "ruleId1", "level1").WithSuppression([]*sarif.Suppression{sarif.NewSuppression("")}), + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2").WithSuppression([]*sarif.Suppression{sarif.NewSuppression("")}), + }, + expectedOutput: []*sarif.Result{}, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, excludeSuppressResults(test.sarifResults)) + } +} + +func TestAddScoreToRunRules(t *testing.T) { + + tests := []struct { + name string + sarifRun *sarif.Run + expectedOutput []*sarif.ReportingDescriptor + }{ + { + sarifRun: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file1", 0, 0, 0, 0, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file2", 0, 0, 0, 0, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule2", "warning"), + ), + expectedOutput: []*sarif.ReportingDescriptor{ + sarif.NewRule("rule1").WithProperties(sarif.Properties{"security-severity": "6.9"}), + sarif.NewRule("rule2").WithProperties(sarif.Properties{"security-severity": "6.9"}), + }, + }, + { + sarifRun: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule1", "none"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule2", "note"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule3", "info"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule4", "warning"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule5", "error"), + ), + expectedOutput: []*sarif.ReportingDescriptor{ + sarif.NewRule("rule1").WithProperties(sarif.Properties{"security-severity": "0.0"}), + sarif.NewRule("rule2").WithProperties(sarif.Properties{"security-severity": "3.9"}), + sarif.NewRule("rule3").WithProperties(sarif.Properties{"security-severity": "6.9"}), + sarif.NewRule("rule4").WithProperties(sarif.Properties{"security-severity": "6.9"}), + sarif.NewRule("rule5").WithProperties(sarif.Properties{"security-severity": "8.9"}), + }, + }, + } + + for _, test := range tests { + addScoreToRunRules(test.sarifRun) + assert.Equal(t, test.expectedOutput, test.sarifRun.Tool.Driver.Rules) + } +} diff --git a/xray/commands/audit/jas/commons_test.go b/xray/commands/audit/jas/commons_test.go new file mode 100644 index 000000000..c43cf94ab --- /dev/null +++ b/xray/commands/audit/jas/commons_test.go @@ -0,0 +1,129 @@ +package jas + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" + clientTestUtils "github.com/jfrog/jfrog-client-go/utils/tests" + + "github.com/jfrog/jfrog-cli-core/v2/xray/utils" + "github.com/stretchr/testify/assert" +) + +var createJFrogAppsConfigCases = []struct { + workingDirs []string +}{ + {workingDirs: []string{}}, + {workingDirs: []string{"working-dir"}}, + {workingDirs: []string{"working-dir-1", "working-dir-2"}}, +} + +func TestCreateJFrogAppsConfig(t *testing.T) { + wd, err := os.Getwd() + assert.NoError(t, err) + + for _, testCase := range createJFrogAppsConfigCases { + t.Run(fmt.Sprintf("%v", testCase.workingDirs), func(t *testing.T) { + jfrogAppsConfig, err := createJFrogAppsConfig(testCase.workingDirs) + assert.NoError(t, err) + assert.NotNil(t, jfrogAppsConfig) + if len(testCase.workingDirs) == 0 { + assert.Len(t, jfrogAppsConfig.Modules, 1) + assert.Equal(t, wd, jfrogAppsConfig.Modules[0].SourceRoot) + return + } + assert.Len(t, jfrogAppsConfig.Modules, len(testCase.workingDirs)) + for i, workingDir := range testCase.workingDirs { + assert.Equal(t, filepath.Join(wd, workingDir), jfrogAppsConfig.Modules[i].SourceRoot) + } + }) + } +} + +func TestCreateJFrogAppsConfigWithConfig(t *testing.T) { + wd, err := os.Getwd() + assert.NoError(t, err) + chdirCallback := clientTestUtils.ChangeDirWithCallback(t, wd, "testdata") + defer chdirCallback() + + jfrogAppsConfig, err := createJFrogAppsConfig([]string{}) + assert.NoError(t, err) + assert.NotNil(t, jfrogAppsConfig) + assert.Equal(t, "1.0", jfrogAppsConfig.Version) + assert.Len(t, jfrogAppsConfig.Modules, 1) +} + +func TestShouldSkipScanner(t *testing.T) { + module := jfrogappsconfig.Module{} + assert.False(t, ShouldSkipScanner(module, utils.IaC)) + + module = jfrogappsconfig.Module{ExcludeScanners: []string{"sast"}} + assert.False(t, ShouldSkipScanner(module, utils.IaC)) + assert.True(t, ShouldSkipScanner(module, utils.Sast)) +} + +var getSourceRootsCases = []struct { + scanner *jfrogappsconfig.Scanner +}{ + {scanner: nil}, + {&jfrogappsconfig.Scanner{WorkingDirs: []string{"working-dir"}}}, + {&jfrogappsconfig.Scanner{WorkingDirs: []string{"working-dir-1", "working-dir-2"}}}, +} + +func TestGetSourceRoots(t *testing.T) { + testGetSourceRoots(t, "source-root") +} + +func TestGetSourceRootsEmptySourceRoot(t *testing.T) { + testGetSourceRoots(t, "") +} + +func testGetSourceRoots(t *testing.T, sourceRoot string) { + sourceRoot, err := filepath.Abs(sourceRoot) + assert.NoError(t, err) + module := jfrogappsconfig.Module{SourceRoot: sourceRoot} + for _, testCase := range getSourceRootsCases { + t.Run("", func(t *testing.T) { + scanner := testCase.scanner + actualSourceRoots, err := GetSourceRoots(module, scanner) + assert.NoError(t, err) + if scanner == nil { + assert.ElementsMatch(t, []string{module.SourceRoot}, actualSourceRoots) + return + } + expectedWorkingDirs := []string{} + for _, workingDir := range scanner.WorkingDirs { + expectedWorkingDirs = append(expectedWorkingDirs, filepath.Join(module.SourceRoot, workingDir)) + } + assert.ElementsMatch(t, actualSourceRoots, expectedWorkingDirs) + }) + } +} + +var getExcludePatternsCases = []struct { + scanner *jfrogappsconfig.Scanner +}{ + {scanner: nil}, + {&jfrogappsconfig.Scanner{WorkingDirs: []string{"exclude-dir"}}}, + {&jfrogappsconfig.Scanner{WorkingDirs: []string{"exclude-dir-1", "exclude-dir-2"}}}, +} + +func TestGetExcludePatterns(t *testing.T) { + module := jfrogappsconfig.Module{ExcludePatterns: []string{"exclude-root"}} + for _, testCase := range getExcludePatternsCases { + t.Run("", func(t *testing.T) { + scanner := testCase.scanner + actualExcludePatterns := GetExcludePatterns(module, scanner) + if scanner == nil { + assert.ElementsMatch(t, module.ExcludePatterns, actualExcludePatterns) + return + } + expectedExcludePatterns := module.ExcludePatterns + expectedExcludePatterns = append(expectedExcludePatterns, scanner.ExcludePatterns...) + assert.ElementsMatch(t, actualExcludePatterns, expectedExcludePatterns) + }) + } +} diff --git a/xray/commands/audit/jas/iac/iacscanner.go b/xray/commands/audit/jas/iac/iacscanner.go index c4ccfdd39..402557e3e 100644 --- a/xray/commands/audit/jas/iac/iacscanner.go +++ b/xray/commands/audit/jas/iac/iacscanner.go @@ -1,17 +1,20 @@ package iac import ( - "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "path/filepath" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" + "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" + "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/owenrumney/go-sarif/v2/sarif" ) const ( - iacScannerType = "iac-scan-modules" - iacScanCommand = "iac" + iacScannerType = "iac-scan-modules" + iacScanCommand = "iac" + iacDocsUrlSuffix = "infrastructure-as-code-iac" ) type IacScanManager struct { @@ -48,15 +51,17 @@ func newIacScanManager(scanner *jas.JasScanner) (manager *IacScanManager) { } } -func (iac *IacScanManager) Run(wd string) (err error) { - scanner := iac.scanner - if err = iac.createConfigFile(wd); err != nil { +func (iac *IacScanManager) Run(module jfrogappsconfig.Module) (err error) { + if jas.ShouldSkipScanner(module, utils.IaC) { + return + } + if err = iac.createConfigFile(module); err != nil { return } if err = iac.runAnalyzerManager(); err != nil { return } - workingDirResults, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, wd) + workingDirResults, err := jas.ReadJasScanRunsFromFile(iac.scanner.ResultsFileName, module.SourceRoot, iacDocsUrlSuffix) if err != nil { return } @@ -75,18 +80,22 @@ type iacScanConfiguration struct { SkippedDirs []string `yaml:"skipped-folders"` } -func (iac *IacScanManager) createConfigFile(currentWd string) error { +func (iac *IacScanManager) createConfigFile(module jfrogappsconfig.Module) error { + roots, err := jas.GetSourceRoots(module, module.Scanners.Iac) + if err != nil { + return err + } configFileContent := iacScanConfig{ Scans: []iacScanConfiguration{ { - Roots: []string{currentWd}, + Roots: roots, Output: iac.scanner.ResultsFileName, Type: iacScannerType, - SkippedDirs: jas.SkippedDirs, + SkippedDirs: jas.GetExcludePatterns(module, module.Scanners.Iac), }, }, } - return jas.CreateScannersConfigFile(iac.scanner.ConfigFileName, configFileContent) + return jas.CreateScannersConfigFile(iac.scanner.ConfigFileName, configFileContent, utils.IaC) } func (iac *IacScanManager) runAnalyzerManager() error { diff --git a/xray/commands/audit/jas/iac/iacscanner_test.go b/xray/commands/audit/jas/iac/iacscanner_test.go index a2332421d..1f9482ded 100644 --- a/xray/commands/audit/jas/iac/iacscanner_test.go +++ b/xray/commands/audit/jas/iac/iacscanner_test.go @@ -1,11 +1,13 @@ package iac import ( - "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "os" "path/filepath" "testing" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" + "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/stretchr/testify/assert" ) @@ -20,7 +22,7 @@ func TestNewIacScanManager(t *testing.T) { if assert.NotNil(t, iacScanManager) { assert.NotEmpty(t, iacScanManager.scanner.ConfigFileName) assert.NotEmpty(t, iacScanManager.scanner.ResultsFileName) - assert.NotEmpty(t, iacScanManager.scanner.WorkingDirs) + assert.NotEmpty(t, iacScanManager.scanner.JFrogAppsConfig.Modules[0].SourceRoot) assert.Equal(t, &jas.FakeServerDetails, iacScanManager.scanner.ServerDetails) } } @@ -33,7 +35,7 @@ func TestIacScan_CreateConfigFile_VerifyFileWasCreated(t *testing.T) { currWd, err := coreutils.GetWorkingDirectory() assert.NoError(t, err) - err = iacScanManager.createConfigFile(currWd) + err = iacScanManager.createConfigFile(jfrogappsconfig.Module{SourceRoot: currWd}) defer func() { err = os.Remove(iacScanManager.scanner.ConfigFileName) @@ -57,7 +59,7 @@ func TestIacParseResults_EmptyResults(t *testing.T) { // Act var err error - iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, iacDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, iacScanManager.iacScannerResults) { assert.Len(t, iacScanManager.iacScannerResults, 1) assert.Empty(t, iacScanManager.iacScannerResults[0].Results) @@ -73,7 +75,7 @@ func TestIacParseResults_ResultsContainIacViolations(t *testing.T) { // Act var err error - iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, iacDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, iacScanManager.iacScannerResults) { assert.Len(t, iacScanManager.iacScannerResults, 1) assert.Len(t, iacScanManager.iacScannerResults[0].Results, 4) diff --git a/xray/commands/audit/jas/sast/sastscanner.go b/xray/commands/audit/jas/sast/sastscanner.go index 35211396f..578bc6775 100644 --- a/xray/commands/audit/jas/sast/sastscanner.go +++ b/xray/commands/audit/jas/sast/sastscanner.go @@ -1,6 +1,11 @@ package sast import ( + "fmt" + + "path/filepath" + + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -9,7 +14,9 @@ import ( ) const ( - sastScanCommand = "zd" + sastScannerType = "sast" + sastScanCommand = "zd" + sastDocsUrlSuffix = "sast" ) type SastScanManager struct { @@ -38,27 +45,69 @@ func newSastScanManager(scanner *jas.JasScanner) (manager *SastScanManager) { } } -func (ssm *SastScanManager) Run(wd string) (err error) { +func (ssm *SastScanManager) Run(module jfrogappsconfig.Module) (err error) { + if jas.ShouldSkipScanner(module, utils.Sast) { + return + } + if err = ssm.createConfigFile(module); err != nil { + return + } scanner := ssm.scanner - if err = ssm.runAnalyzerManager(wd); err != nil { + if err = ssm.runAnalyzerManager(filepath.Dir(ssm.scanner.AnalyzerManager.AnalyzerManagerFullPath)); err != nil { return } - workingDirRuns, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, wd) + workingDirRuns, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, module.SourceRoot, sastDocsUrlSuffix) if err != nil { return } - ssm.sastScannerResults = append(ssm.sastScannerResults, groupResultsByLocation(workingDirRuns)...) + groupResultsByLocation(workingDirRuns) + ssm.sastScannerResults = append(ssm.sastScannerResults, workingDirRuns...) return } +type sastScanConfig struct { + Scans []scanConfiguration `yaml:"scans,omitempty"` +} + +type scanConfiguration struct { + Roots []string `yaml:"roots,omitempty"` + Type string `yaml:"type,omitempty"` + Language string `yaml:"language,omitempty"` + ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` + ExcludedRules []string `yaml:"excluded-rules,omitempty"` +} + +func (ssm *SastScanManager) createConfigFile(module jfrogappsconfig.Module) error { + sastScanner := module.Scanners.Sast + if sastScanner == nil { + sastScanner = &jfrogappsconfig.SastScanner{} + } + roots, err := jas.GetSourceRoots(module, &sastScanner.Scanner) + if err != nil { + return err + } + configFileContent := sastScanConfig{ + Scans: []scanConfiguration{ + { + Type: sastScannerType, + Roots: roots, + Language: sastScanner.Language, + ExcludedRules: sastScanner.ExcludedRules, + ExcludePatterns: jas.GetExcludePatterns(module, &sastScanner.Scanner), + }, + }, + } + return jas.CreateScannersConfigFile(ssm.scanner.ConfigFileName, configFileContent, utils.Sast) +} + func (ssm *SastScanManager) runAnalyzerManager(wd string) error { - return ssm.scanner.AnalyzerManager.Exec(ssm.scanner.ResultsFileName, sastScanCommand, wd, ssm.scanner.ServerDetails) + return ssm.scanner.AnalyzerManager.ExecWithOutputFile(ssm.scanner.ConfigFileName, sastScanCommand, wd, ssm.scanner.ResultsFileName, ssm.scanner.ServerDetails) } // In the Sast scanner, there can be multiple results with the same location. // The only difference is that their CodeFlow values are different. // We combine those under the same result location value -func groupResultsByLocation(sarifRuns []*sarif.Run) []*sarif.Run { +func groupResultsByLocation(sarifRuns []*sarif.Run) { for _, sastRun := range sarifRuns { locationToResult := map[string]*sarif.Result{} for _, sastResult := range sastRun.Results { @@ -71,25 +120,28 @@ func groupResultsByLocation(sarifRuns []*sarif.Run) []*sarif.Run { } sastRun.Results = maps.Values(locationToResult) } - return sarifRuns } -// In Sast there is only one location for each result -func getResultFileName(result *sarif.Result) string { - if len(result.Locations) > 0 { - return utils.GetLocationFileName(result.Locations[0]) +func getResultLocationStr(result *sarif.Result) string { + if len(result.Locations) == 0 { + return "" } - return "" + location := result.Locations[0] + return fmt.Sprintf("%s%d%d%d%d", + utils.GetLocationFileName(location), + utils.GetLocationStartLine(location), + utils.GetLocationStartColumn(location), + utils.GetLocationEndLine(location), + utils.GetLocationEndColumn(location)) } -// In Sast there is only one location for each result -func getResultStartLocationInFile(result *sarif.Result) string { - if len(result.Locations) > 0 { - return utils.GetStartLocationInFile(result.Locations[0]) +func getResultRuleId(result *sarif.Result) string { + if result.RuleID == nil { + return "" } - return "" + return *result.RuleID } func getResultId(result *sarif.Result) string { - return getResultFileName(result) + getResultStartLocationInFile(result) + utils.GetResultSeverity(result) + utils.GetResultMsgText(result) + return getResultRuleId(result) + utils.GetResultSeverity(result) + utils.GetResultMsgText(result) + getResultLocationStr(result) } diff --git a/xray/commands/audit/jas/sast/sastscanner_test.go b/xray/commands/audit/jas/sast/sastscanner_test.go index 66c423403..43eedf705 100644 --- a/xray/commands/audit/jas/sast/sastscanner_test.go +++ b/xray/commands/audit/jas/sast/sastscanner_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" + "github.com/jfrog/jfrog-cli-core/v2/xray/utils" + "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" ) @@ -19,7 +21,7 @@ func TestNewSastScanManager(t *testing.T) { if assert.NotNil(t, sastScanManager) { assert.NotEmpty(t, sastScanManager.scanner.ConfigFileName) assert.NotEmpty(t, sastScanManager.scanner.ResultsFileName) - assert.NotEmpty(t, sastScanManager.scanner.WorkingDirs) + assert.NotEmpty(t, sastScanManager.scanner.JFrogAppsConfig.Modules[0].SourceRoot) assert.Equal(t, &jas.FakeServerDetails, sastScanManager.scanner.ServerDetails) } } @@ -34,13 +36,13 @@ func TestSastParseResults_EmptyResults(t *testing.T) { // Act var err error - sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, sastDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { assert.Len(t, sastScanManager.sastScannerResults, 1) assert.Empty(t, sastScanManager.sastScannerResults[0].Results) - sastScanManager.sastScannerResults = groupResultsByLocation(sastScanManager.sastScannerResults) + groupResultsByLocation(sastScanManager.sastScannerResults) assert.Len(t, sastScanManager.sastScannerResults, 1) assert.Empty(t, sastScanManager.sastScannerResults[0].Results) } @@ -55,14 +57,100 @@ func TestSastParseResults_ResultsContainIacViolations(t *testing.T) { // Act var err error - sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, sastDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { assert.Len(t, sastScanManager.sastScannerResults, 1) assert.NotEmpty(t, sastScanManager.sastScannerResults[0].Results) - sastScanManager.sastScannerResults = groupResultsByLocation(sastScanManager.sastScannerResults) + groupResultsByLocation(sastScanManager.sastScannerResults) // File has 4 results, 2 of them at the same location different codeFlow assert.Len(t, sastScanManager.sastScannerResults[0].Results, 3) } } + +func TestGroupResultsByLocation(t *testing.T) { + tests := []struct { + run *sarif.Run + expectedOutput *sarif.Run + }{ + { + run: utils.CreateRunWithDummyResults(), + expectedOutput: utils.CreateRunWithDummyResults(), + }, + { + // No similar groups at all + run: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "note"), + utils.CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file2", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other", 0, 0, 0, 0, "other-snippet"), + utils.CreateLocation("file2", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file2", 1, 2, 3, 4, "snippet", "rule2", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other2", 1, 1, 1, 1, "other-snippet2"), + utils.CreateLocation("file2", 1, 2, 3, 4, "snippet"), + )), + }), + ), + expectedOutput: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "note"), + utils.CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file2", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other", 0, 0, 0, 0, "other-snippet"), + utils.CreateLocation("file2", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file2", 1, 2, 3, 4, "snippet", "rule2", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other2", 1, 1, 1, 1, "other-snippet2"), + utils.CreateLocation("file2", 1, 2, 3, 4, "snippet"), + )), + }), + ), + }, + { + // With similar groups + run: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other", 0, 0, 0, 0, "other-snippet"), + utils.CreateLocation("file", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other2", 1, 1, 1, 1, "other-snippet"), + utils.CreateLocation("file", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info"), + ), + expectedOutput: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other", 0, 0, 0, 0, "other-snippet"), + utils.CreateLocation("file", 1, 2, 3, 4, "snippet"), + )), + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other2", 1, 1, 1, 1, "other-snippet"), + utils.CreateLocation("file", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet", "rule1", "info"), + ), + }, + } + + for _, test := range tests { + groupResultsByLocation([]*sarif.Run{test.run}) + assert.ElementsMatch(t, test.expectedOutput.Results, test.run.Results) + } +} diff --git a/xray/commands/audit/jas/secrets/secretsscanner.go b/xray/commands/audit/jas/secrets/secretsscanner.go index cf5df05f8..aad7fc6a6 100644 --- a/xray/commands/audit/jas/secrets/secretsscanner.go +++ b/xray/commands/audit/jas/secrets/secretsscanner.go @@ -4,6 +4,7 @@ import ( "path/filepath" "strings" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -11,8 +12,9 @@ import ( ) const ( - secretsScanCommand = "sec" - secretsScannerType = "secrets-scan" + secretsScanCommand = "sec" + secretsScannerType = "secrets-scan" + secretsDocsUrlSuffix = "secrets" ) type SecretScanManager struct { @@ -48,19 +50,21 @@ func newSecretsScanManager(scanner *jas.JasScanner) (manager *SecretScanManager) } } -func (s *SecretScanManager) Run(wd string) (err error) { - scanner := s.scanner - if err = s.createConfigFile(wd); err != nil { +func (ssm *SecretScanManager) Run(module jfrogappsconfig.Module) (err error) { + if jas.ShouldSkipScanner(module, utils.Secrets) { return } - if err = s.runAnalyzerManager(); err != nil { + if err = ssm.createConfigFile(module); err != nil { return } - workingDirRuns, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, wd) + if err = ssm.runAnalyzerManager(); err != nil { + return + } + workingDirRuns, err := jas.ReadJasScanRunsFromFile(ssm.scanner.ResultsFileName, module.SourceRoot, secretsDocsUrlSuffix) if err != nil { return } - s.secretsScannerResults = append(s.secretsScannerResults, processSecretScanRuns(workingDirRuns)...) + ssm.secretsScannerResults = append(ssm.secretsScannerResults, processSecretScanRuns(workingDirRuns)...) return } @@ -75,18 +79,22 @@ type secretsScanConfiguration struct { SkippedDirs []string `yaml:"skipped-folders"` } -func (s *SecretScanManager) createConfigFile(currentWd string) error { +func (s *SecretScanManager) createConfigFile(module jfrogappsconfig.Module) error { + roots, err := jas.GetSourceRoots(module, module.Scanners.Secrets) + if err != nil { + return err + } configFileContent := secretsScanConfig{ Scans: []secretsScanConfiguration{ { - Roots: []string{currentWd}, + Roots: roots, Output: s.scanner.ResultsFileName, Type: secretsScannerType, - SkippedDirs: jas.SkippedDirs, + SkippedDirs: jas.GetExcludePatterns(module, module.Scanners.Secrets), }, }, } - return jas.CreateScannersConfigFile(s.scanner.ConfigFileName, configFileContent) + return jas.CreateScannersConfigFile(s.scanner.ConfigFileName, configFileContent, utils.Secrets) } func (s *SecretScanManager) runAnalyzerManager() error { @@ -105,8 +113,7 @@ func processSecretScanRuns(sarifRuns []*sarif.Run) []*sarif.Run { // Hide discovered secrets value for _, secretResult := range secretRun.Results { for _, location := range secretResult.Locations { - secret := utils.GetLocationSnippetPointer(location) - utils.SetLocationSnippet(location, maskSecret(*secret)) + utils.SetLocationSnippet(location, maskSecret(utils.GetLocationSnippet(location))) } } } diff --git a/xray/commands/audit/jas/secrets/secretsscanner_test.go b/xray/commands/audit/jas/secrets/secretsscanner_test.go index 14e917e16..1b2a053ec 100644 --- a/xray/commands/audit/jas/secrets/secretsscanner_test.go +++ b/xray/commands/audit/jas/secrets/secretsscanner_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -29,7 +30,7 @@ func TestSecretsScan_CreateConfigFile_VerifyFileWasCreated(t *testing.T) { currWd, err := coreutils.GetWorkingDirectory() assert.NoError(t, err) - err = secretScanManager.createConfigFile(currWd) + err = secretScanManager.createConfigFile(jfrogappsconfig.Module{SourceRoot: currWd}) assert.NoError(t, err) defer func() { @@ -65,7 +66,7 @@ func TestParseResults_EmptyResults(t *testing.T) { // Act var err error - secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, secretsDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, secretScanManager.secretsScannerResults) { @@ -88,7 +89,7 @@ func TestParseResults_ResultsContainSecrets(t *testing.T) { // Act var err error - secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, secretsDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, secretScanManager.secretsScannerResults) { diff --git a/xray/commands/audit/jas/testdata/.jfrog/jfrog-apps-config.yml b/xray/commands/audit/jas/testdata/.jfrog/jfrog-apps-config.yml new file mode 100644 index 000000000..9357059d7 --- /dev/null +++ b/xray/commands/audit/jas/testdata/.jfrog/jfrog-apps-config.yml @@ -0,0 +1,50 @@ +# [Required] JFrog Applications Config version +version: "1.0" + +modules: + # [Required] Module name + - name: FrogLeapApp + # [Optional, default: "."] Application's root directory + source_root: "src" + # [Optional] Directories to exclude from scanning across all scanners + exclude_patterns: + - "docs/" + # [Optional] Scanners to exclude from JFrog Advanced Security (Options: "secrets", "sast", "iac") + exclude_scanners: + - secrets + # [Optional] Customize scanner configurations + scanners: + # [Optional] Configuration for Static Application Security Testing (SAST) + sast: + # [Optional] Specify the programming language for SAST + language: java + # [Optional] Working directories specific to SAST (Relative to source_root) + working_dirs: + - "src/module1" + - "src/module2" + # [Optional] Additional exclude patterns for this scanner + exclude_patterns: + - "src/module1/test" + # [Optional] List of specific scan rules to exclude from the scan + excluded_rules: + - xss-injection + + # [Optional] Configuration for secrets scan + secrets: + # [Optional] Working directories specific to the secret scanner (Relative to source_root) + working_dirs: + - "src/module1" + - "src/module2" + # [Optional] Additional exclude patterns for this scanner + exclude_patterns: + - "src/module1/test" + + # [Optional] Configuration for Infrastructure as Code scan (IaC) + iac: + # [Optional] Working directories specific to IaC (Relative to source_root) + working_dirs: + - "src/module1" + - "src/module2" + # [Optional] Additional exclude patterns for this Scanner + exclude_patterns: + - "src/module1/test" \ No newline at end of file diff --git a/xray/commands/audit/jasrunner.go b/xray/commands/audit/jasrunner.go index 76a59ff8e..c528fdadc 100644 --- a/xray/commands/audit/jasrunner.go +++ b/xray/commands/audit/jasrunner.go @@ -52,9 +52,6 @@ func runJasScannersAndSetResults(scanResults *utils.ExtendedScanResults, directD if err != nil { return } - if !utils.IsSastSupported() { - return - } if progress != nil { progress.SetHeadlineMsg("Running SAST scanning") } diff --git a/xray/commands/audit/sca/java/gradle-dep-tree.jar b/xray/commands/audit/sca/java/gradle-dep-tree.jar index 2762a71ba..532445bc1 100644 Binary files a/xray/commands/audit/sca/java/gradle-dep-tree.jar and b/xray/commands/audit/sca/java/gradle-dep-tree.jar differ diff --git a/xray/commands/audit/sca/java/gradle.go b/xray/commands/audit/sca/java/gradle.go index d140efffd..933788192 100644 --- a/xray/commands/audit/sca/java/gradle.go +++ b/xray/commands/audit/sca/java/gradle.go @@ -2,11 +2,8 @@ package java import ( _ "embed" - "encoding/base64" - "encoding/json" "errors" "fmt" - "github.com/jfrog/gofrog/datastructures" "os" "os/exec" "path/filepath" @@ -58,60 +55,12 @@ allprojects { var gradleDepTreeJar []byte type depTreeManager struct { - dependenciesTree server *config.ServerDetails releasesRepo string depsRepo string useWrapper bool } -// dependenciesTree represents a map between dependencies to their children dependencies in multiple projects. -type dependenciesTree struct { - tree map[string][]dependenciesPaths -} - -// dependenciesPaths represents a map between dependencies to their children dependencies in a single project. -type dependenciesPaths struct { - Paths map[string]dependenciesPaths `json:"children"` -} - -// The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project. -// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in dependenciesTree struct. -func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { - outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n") - for _, path := range outputFiles { - tree, err := os.ReadFile(strings.TrimSpace(path)) - if err != nil { - return errorutils.CheckError(err) - } - - encodedFileName := path[strings.LastIndex(path, string(os.PathSeparator))+1:] - decodedFileName, err := base64.StdEncoding.DecodeString(encodedFileName) - if err != nil { - return errorutils.CheckError(err) - } - - if err = dtp.appendDependenciesPaths(tree, string(decodedFileName)); err != nil { - return errorutils.CheckError(err) - } - } - return nil -} - -func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte, fileName string) error { - var deps dependenciesPaths - if err := json.Unmarshal(jsonDepTree, &deps); err != nil { - return errorutils.CheckError(err) - } - if dtp.tree == nil { - dtp.tree = make(map[string][]dependenciesPaths) - } - if len(deps.Paths) > 0 { - dtp.tree[fileName] = append(dtp.tree[fileName], deps) - } - return nil -} - func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, err error) { manager := &depTreeManager{useWrapper: params.UseWrapper} if params.IgnoreConfigFile { @@ -130,7 +79,7 @@ func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []* if err != nil { return } - dependencyTree, uniqueDeps, err = manager.getGraphFromDepTree(outputFileContent) + dependencyTree, uniqueDeps, err = getGraphFromDepTree(outputFileContent) return } @@ -163,7 +112,7 @@ func (dtp *depTreeManager) createDepTreeScriptAndGetDir() (tmpDir string, err er if err != nil { return } - gradleDepTreeJarPath := filepath.Join(tmpDir, string(gradleDepTreeJarFile)) + gradleDepTreeJarPath := filepath.Join(tmpDir, gradleDepTreeJarFile) if err = errorutils.CheckError(os.WriteFile(gradleDepTreeJarPath, gradleDepTreeJar, 0666)); err != nil { return } @@ -237,42 +186,6 @@ func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileConte return } -// Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input -func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*xrayUtils.GraphNode, []string, error) { - if err := dtp.parseDepTreeFiles(outputFileContent); err != nil { - return nil, nil, err - } - var depsGraph []*xrayUtils.GraphNode - uniqueDepsSet := datastructures.MakeSet[string]() - for dependency, children := range dtp.tree { - directDependency := &xrayUtils.GraphNode{ - Id: GavPackageTypeIdentifier + dependency, - Nodes: []*xrayUtils.GraphNode{}, - } - for _, childPath := range children { - populateGradleDependencyTree(directDependency, childPath, uniqueDepsSet) - } - depsGraph = append(depsGraph, directDependency) - } - return depsGraph, uniqueDepsSet.ToSlice(), nil -} - -func populateGradleDependencyTree(currNode *xrayUtils.GraphNode, currNodeChildren dependenciesPaths, uniqueDepsSet *datastructures.Set[string]) { - uniqueDepsSet.Add(currNode.Id) - for gav, children := range currNodeChildren.Paths { - childNode := &xrayUtils.GraphNode{ - Id: GavPackageTypeIdentifier + gav, - Nodes: []*xrayUtils.GraphNode{}, - Parent: currNode, - } - if currNode.NodeHasLoop() { - return - } - populateGradleDependencyTree(childNode, children, uniqueDepsSet) - currNode.Nodes = append(currNode.Nodes, childNode) - } -} - func getDepTreeArtifactoryRepository(remoteRepo string, server *config.ServerDetails) (string, error) { if remoteRepo == "" || server.IsEmpty() { return "", nil diff --git a/xray/commands/audit/sca/java/gradle_test.go b/xray/commands/audit/sca/java/gradle_test.go index e9d09b742..27c4aaaa0 100644 --- a/xray/commands/audit/sca/java/gradle_test.go +++ b/xray/commands/audit/sca/java/gradle_test.go @@ -47,10 +47,10 @@ func TestGradleTreesWithoutConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{}) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, uniqueDeps, 11) - assert.Len(t, modulesDependencyTrees, 2) + assert.Len(t, uniqueDeps, 9) + assert.Len(t, modulesDependencyTrees, 5) // Check module - module := sca.GetAndAssertNode(t, modulesDependencyTrees, "webservice") + module := sca.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.example.gradle:webservice:1.0") assert.Len(t, module.Nodes, 7) // Check direct dependency @@ -71,10 +71,10 @@ func TestGradleTreesWithConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 3) - assert.Len(t, uniqueDeps, 11) + assert.Len(t, modulesDependencyTrees, 5) + assert.Len(t, uniqueDeps, 8) // Check module - module := sca.GetAndAssertNode(t, modulesDependencyTrees, "api") + module := sca.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test.gradle.publish:api:1.0-SNAPSHOT") assert.Len(t, module.Nodes, 4) // Check direct dependency @@ -86,22 +86,6 @@ func TestGradleTreesWithConfig(t *testing.T) { } } -func TestGradleTreesExcludeTestDeps(t *testing.T) { - // Create and change directory to test workspace - tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server") - defer cleanUp() - assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) - - // Run getModulesDependencyTrees - modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) - if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 2) - assert.Len(t, uniqueDeps, 11) - // Check direct dependency - assert.Nil(t, sca.GetModule(modulesDependencyTrees, "services")) - } -} - func TestIsGradleWrapperExist(t *testing.T) { // Check Gradle wrapper doesn't exist isWrapperExist, err := isGradleWrapperExist() @@ -168,67 +152,6 @@ func TestGetDepTreeArtifactoryRepository(t *testing.T) { } } -func TestGetGraphFromDepTree(t *testing.T) { - // Create and change directory to test workspace - tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server") - defer func() { - cleanUp() - }() - assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) - testCase := struct { - name string - expectedTree map[string]map[string]string - expectedUniqueDeps []string - }{ - name: "ValidOutputFileContent", - expectedTree: map[string]map[string]string{ - GavPackageTypeIdentifier + "shared": {}, - GavPackageTypeIdentifier + filepath.Base(tempDirPath): {}, - GavPackageTypeIdentifier + "services": {}, - GavPackageTypeIdentifier + "webservice": { - GavPackageTypeIdentifier + "junit:junit:4.11": "", - GavPackageTypeIdentifier + "commons-io:commons-io:1.2": "", - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": "", - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", - GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": "", - }, - GavPackageTypeIdentifier + "api": { - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", - }, - }, - expectedUniqueDeps: []string{ - GavPackageTypeIdentifier + "webservice", - GavPackageTypeIdentifier + "junit:junit:4.11", - GavPackageTypeIdentifier + "commons-io:commons-io:1.2", - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0", - GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2", - GavPackageTypeIdentifier + "api", - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4", - GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3", - GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2", - }, - } - - manager := &depTreeManager{} - outputFileContent, err := manager.runGradleDepTree() - assert.NoError(t, err) - depTree, uniqueDeps, err := (&depTreeManager{}).getGraphFromDepTree(outputFileContent) - assert.NoError(t, err) - assert.ElementsMatch(t, uniqueDeps, testCase.expectedUniqueDeps, "First is actual, Second is Expected") - - for _, dependency := range depTree { - depChild, exists := testCase.expectedTree[dependency.Id] - assert.True(t, exists) - assert.Equal(t, len(depChild), len(dependency.Nodes)) - } -} - func TestCreateDepTreeScript(t *testing.T) { manager := &depTreeManager{} tmpDir, err := manager.createDepTreeScriptAndGetDir() diff --git a/xray/commands/audit/sca/java/javautils.go b/xray/commands/audit/sca/java/javautils.go index 9653de43f..942066965 100644 --- a/xray/commands/audit/sca/java/javautils.go +++ b/xray/commands/audit/sca/java/javautils.go @@ -1,12 +1,15 @@ package java import ( + "encoding/json" "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" + "os" "strconv" + "strings" "time" buildinfo "github.com/jfrog/build-info-go/entities" @@ -58,7 +61,7 @@ func createGavDependencyTree(buildConfig *artifactoryUtils.BuildConfiguration) ( if len(generatedBuildsInfos) == 0 { return nil, nil, errorutils.CheckErrorf("Couldn't find build " + buildName + "/" + buildNumber) } - modules := []*xrayUtils.GraphNode{} + var modules []*xrayUtils.GraphNode uniqueDepsSet := datastructures.MakeSet[string]() for _, module := range generatedBuildsInfos[0].Modules { modules = append(modules, addModuleTree(module, uniqueDepsSet)) @@ -133,7 +136,7 @@ func hasLoop(idsAdded []string, idToAdd string) bool { return false } -func BuildDependencyTree(params *xrayutils.AuditBasicParams, tech coreutils.Technology) ([]*xrayUtils.GraphNode, []string, error) { +func BuildDependencyTree(params xrayutils.AuditParams, tech coreutils.Technology) ([]*xrayUtils.GraphNode, []string, error) { serverDetails, err := params.ServerDetails() if err != nil { return nil, nil, err @@ -173,3 +176,73 @@ func (dm *dependencyMultimap) putChild(parent string, child *buildinfo.Dependenc func (dm *dependencyMultimap) getChildren(parent string) map[string]*buildinfo.Dependency { return dm.multimap[parent] } + +// The structure of a dependency tree of a module in a Gradle/Maven project, as created by the gradle-dep-tree and maven-dep-tree plugins. +type moduleDepTree struct { + Root string `json:"root"` + Nodes map[string]depTreeNode `json:"nodes"` +} + +type depTreeNode struct { + Children []string `json:"children"` +} + +// getGraphFromDepTree reads the output files of the gradle-dep-tree and maven-dep-tree plugins and returns them as a slice of GraphNodes. +// It takes the output of the plugin's run (which is a byte representation of a list of paths of the output files, separated by newlines) as input. +func getGraphFromDepTree(depTreeOutput []byte) (depsGraph []*xrayUtils.GraphNode, uniqueDeps []string, err error) { + modules, err := parseDepTreeFiles(depTreeOutput) + if err != nil { + return + } + uniqueDepsSet := datastructures.MakeSet[string]() + for _, moduleTree := range modules { + directDependency := &xrayUtils.GraphNode{ + Id: GavPackageTypeIdentifier + moduleTree.Root, + Nodes: []*xrayUtils.GraphNode{}, + } + populateDependencyTree(directDependency, moduleTree.Root, moduleTree, uniqueDepsSet) + depsGraph = append(depsGraph, directDependency) + } + uniqueDeps = uniqueDepsSet.ToSlice() + return +} + +func populateDependencyTree(currNode *xrayUtils.GraphNode, currNodeId string, moduleTree *moduleDepTree, uniqueDepsSet *datastructures.Set[string]) { + if currNode.NodeHasLoop() { + return + } + for _, childId := range moduleTree.Nodes[currNodeId].Children { + childGav := GavPackageTypeIdentifier + childId + childNode := &xrayUtils.GraphNode{ + Id: childGav, + Nodes: []*xrayUtils.GraphNode{}, + Parent: currNode, + } + uniqueDepsSet.Add(childGav) + populateDependencyTree(childNode, childId, moduleTree, uniqueDepsSet) + currNode.Nodes = append(currNode.Nodes, childNode) + } +} + +func parseDepTreeFiles(jsonFilePaths []byte) ([]*moduleDepTree, error) { + outputFilePaths := strings.Split(strings.TrimSpace(string(jsonFilePaths)), "\n") + var modules []*moduleDepTree + for _, path := range outputFilePaths { + results, err := parseDepTreeFile(path) + if err != nil { + return nil, err + } + modules = append(modules, results) + } + return modules, nil +} + +func parseDepTreeFile(path string) (results *moduleDepTree, err error) { + depTreeJson, err := os.ReadFile(strings.TrimSpace(path)) + if errorutils.CheckError(err) != nil { + return + } + results = &moduleDepTree{} + err = errorutils.CheckError(json.Unmarshal(depTreeJson, &results)) + return +} diff --git a/xray/commands/audit/sca/java/javautils_test.go b/xray/commands/audit/sca/java/javautils_test.go new file mode 100644 index 000000000..325a59408 --- /dev/null +++ b/xray/commands/audit/sca/java/javautils_test.go @@ -0,0 +1,68 @@ +package java + +import ( + "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca" + "github.com/stretchr/testify/assert" + "os" + "path/filepath" + "testing" +) + +func TestGetGraphFromDepTree(t *testing.T) { + // Create and change directory to test workspace + tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server") + defer func() { + cleanUp() + }() + assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) + testCase := struct { + name string + expectedTree map[string]map[string]string + expectedUniqueDeps []string + }{ + name: "ValidOutputFileContent", + expectedTree: map[string]map[string]string{ + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:" + filepath.Base(tempDirPath) + ":1.0": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:services:1.0": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:webservice:1.0": { + GavPackageTypeIdentifier + "junit:junit:4.11": "", + GavPackageTypeIdentifier + "commons-io:commons-io:1.2": "", + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": "", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", + GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": "", + }, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": { + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", + }, + }, + expectedUniqueDeps: []string{ + GavPackageTypeIdentifier + "junit:junit:4.11", + GavPackageTypeIdentifier + "commons-io:commons-io:1.2", + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0", + GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4", + GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3", + GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2", + }, + } + + manager := &depTreeManager{} + outputFileContent, err := manager.runGradleDepTree() + assert.NoError(t, err) + depTree, uniqueDeps, err := getGraphFromDepTree(outputFileContent) + assert.NoError(t, err) + assert.ElementsMatch(t, uniqueDeps, testCase.expectedUniqueDeps, "First is actual, Second is Expected") + + for _, dependency := range depTree { + depChild, exists := testCase.expectedTree[dependency.Id] + assert.True(t, exists) + assert.Equal(t, len(depChild), len(dependency.Nodes)) + } +} diff --git a/xray/commands/audit/sca/npm/npm.go b/xray/commands/audit/sca/npm/npm.go index 269c6993e..9c863f877 100644 --- a/xray/commands/audit/sca/npm/npm.go +++ b/xray/commands/audit/sca/npm/npm.go @@ -15,7 +15,7 @@ const ( ignoreScriptsFlag = "--ignore-scripts" ) -func BuildDependencyTree(npmArgs []string) (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps []string, err error) { +func BuildDependencyTree(params utils.AuditParams) (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps []string, err error) { currentDir, err := coreutils.GetWorkingDirectory() if err != nil { return @@ -28,10 +28,11 @@ func BuildDependencyTree(npmArgs []string) (dependencyTrees []*xrayUtils.GraphNo if err != nil { return } - npmArgs = addIgnoreScriptsFlag(npmArgs) + + treeDepsParam := createTreeDepsParam(params) // Calculate npm dependencies - dependenciesMap, err := biutils.CalculateDependenciesMap(npmExecutablePath, currentDir, packageInfo.BuildInfoModuleId(), npmArgs, log.Logger) + dependenciesMap, err := biutils.CalculateDependenciesMap(npmExecutablePath, currentDir, packageInfo.BuildInfoModuleId(), treeDepsParam, log.Logger) if err != nil { log.Info("Used npm version:", npmVersion.GetVersion()) return @@ -46,6 +47,22 @@ func BuildDependencyTree(npmArgs []string) (dependencyTrees []*xrayUtils.GraphNo return } +func createTreeDepsParam(params utils.AuditParams) biutils.NpmTreeDepListParam { + if params == nil { + return biutils.NpmTreeDepListParam{ + Args: addIgnoreScriptsFlag([]string{}), + } + } + npmTreeDepParam := biutils.NpmTreeDepListParam{ + Args: addIgnoreScriptsFlag(params.Args()), + } + if npmParams, ok := params.(utils.AuditNpmParams); ok { + npmTreeDepParam.IgnoreNodeModules = npmParams.NpmIgnoreNodeModules() + npmTreeDepParam.OverwritePackageLock = npmParams.NpmOverwritePackageLock() + } + return npmTreeDepParam +} + // Add the --ignore-scripts to prevent execution of npm scripts during npm install. func addIgnoreScriptsFlag(npmArgs []string) []string { if !slices.Contains(npmArgs, ignoreScriptsFlag) { diff --git a/xray/commands/audit/sca/npm/npm_test.go b/xray/commands/audit/sca/npm/npm_test.go index ff4d0aa01..36871d29a 100644 --- a/xray/commands/audit/sca/npm/npm_test.go +++ b/xray/commands/audit/sca/npm/npm_test.go @@ -116,6 +116,6 @@ func TestIgnoreScripts(t *testing.T) { // The package.json file contain a postinstall script running an "exit 1" command. // Without the "--ignore-scripts" flag, the test will fail. - _, _, err := BuildDependencyTree([]string{}) + _, _, err := BuildDependencyTree(nil) assert.NoError(t, err) } diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index f78ac8dc2..ae4a25b1a 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -133,9 +133,9 @@ func getDirectDependenciesFromTree(dependencyTrees []*xrayCmdUtils.GraphNode) [] return directDependencies.ToSlice() } -func GetTechDependencyTree(params *xrayutils.AuditBasicParams, tech coreutils.Technology) (flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, err error) { +func GetTechDependencyTree(params xrayutils.AuditParams, tech coreutils.Technology) (flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, err error) { logMessage := fmt.Sprintf("Calculating %s dependencies", tech.ToFormal()) - log.Info(logMessage) + log.Info(logMessage + "...") if params.Progress() != nil { params.Progress().SetHeadlineMsg(logMessage) } @@ -149,7 +149,7 @@ func GetTechDependencyTree(params *xrayutils.AuditBasicParams, tech coreutils.Te case coreutils.Maven, coreutils.Gradle: fullDependencyTrees, uniqueDeps, err = java.BuildDependencyTree(params, tech) case coreutils.Npm: - fullDependencyTrees, uniqueDeps, err = npm.BuildDependencyTree(params.Args()) + fullDependencyTrees, uniqueDeps, err = npm.BuildDependencyTree(params) case coreutils.Yarn: fullDependencyTrees, uniqueDeps, err = yarn.BuildDependencyTree() case coreutils.Go: diff --git a/xray/commands/curation/curationaudit.go b/xray/commands/curation/curationaudit.go index 1ef10d1bb..e6bf649c0 100644 --- a/xray/commands/curation/curationaudit.go +++ b/xray/commands/curation/curationaudit.go @@ -111,13 +111,13 @@ type CurationAuditCommand struct { workingDirs []string OriginPath string parallelRequests int - *utils.AuditBasicParams + utils.AuditParams } func NewCurationAuditCommand() *CurationAuditCommand { return &CurationAuditCommand{ extractPoliciesRegex: regexp.MustCompile(extractPoliciesRegexTemplate), - AuditBasicParams: &utils.AuditBasicParams{}, + AuditParams: &utils.AuditBasicParams{}, } } @@ -192,8 +192,17 @@ func (ca *CurationAuditCommand) doCurateAudit(results map[string][]*PackageStatu return nil } +func (ca *CurationAuditCommand) getAuditParamsByTech(tech coreutils.Technology) utils.AuditParams { + if tech == coreutils.Npm { + return utils.AuditNpmParams{AuditParams: ca.AuditParams}. + SetNpmIgnoreNodeModules(true). + SetNpmOverwritePackageLock(true) + } + return ca.AuditParams +} + func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map[string][]*PackageStatus) error { - flattenGraph, fullDependenciesTree, err := audit.GetTechDependencyTree(ca.AuditBasicParams, tech) + flattenGraph, fullDependenciesTree, err := audit.GetTechDependencyTree(ca.getAuditParamsByTech(tech), tech) if err != nil { return err } diff --git a/xray/commands/scan/buildscan.go b/xray/commands/scan/buildscan.go index 7dd6bdcd0..ec9f9a37e 100644 --- a/xray/commands/scan/buildscan.go +++ b/xray/commands/scan/buildscan.go @@ -128,22 +128,30 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: scanResponse} + resultsPrinter := xrutils.NewResultsWriter(extendedScanResults). + SetOutputFormat(bsc.outputFormat). + SetIncludeVulnerabilities(bsc.includeVulnerabilities). + SetIncludeLicenses(false). + SetIsMultipleRootProject(true). + SetPrintExtendedTable(bsc.printExtendedTable). + SetScanType(services.Binary). + SetExtraMessages(nil) + if bsc.outputFormat != xrutils.Table { // Print the violations and/or vulnerabilities as part of one JSON. - err = xrutils.PrintScanResults(extendedScanResults, nil, bsc.outputFormat, false, false, false, bsc.printExtendedTable, true, nil) + err = resultsPrinter.PrintScanResults() } else { // Print two different tables for violations and vulnerabilities (if needed) // If "No Xray Fail build policy...." error received, no need to print violations if !noFailBuildPolicy { - err = xrutils.PrintScanResults(extendedScanResults, nil, bsc.outputFormat, false, false, false, bsc.printExtendedTable, true, nil) - if err != nil { + if err = resultsPrinter.PrintScanResults(); err != nil { return false, err } } if bsc.includeVulnerabilities { - err = xrutils.PrintScanResults(extendedScanResults, nil, bsc.outputFormat, true, false, false, bsc.printExtendedTable, true, nil) - if err != nil { + resultsPrinter.SetIncludeVulnerabilities(true) + if err = resultsPrinter.PrintScanResults(); err != nil { return false, err } } diff --git a/xray/commands/scan/scan.go b/xray/commands/scan/scan.go index b77a3c069..cd910ea0a 100644 --- a/xray/commands/scan/scan.go +++ b/xray/commands/scan/scan.go @@ -242,14 +242,18 @@ func (scanCmd *ScanCommand) Run() (err error) { scanErrors = appendErrorSlice(scanErrors, fileProducerErrors) scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors) extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: flatResults} - err = xrutils.PrintScanResults(extendedScanResults, - scanErrors, - scanCmd.outputFormat, - scanCmd.includeVulnerabilities, - scanCmd.includeLicenses, - true, - scanCmd.printExtendedTable, true, nil, - ) + + if err = xrutils.NewResultsWriter(extendedScanResults). + SetOutputFormat(scanCmd.outputFormat). + SetIncludeVulnerabilities(scanCmd.includeVulnerabilities). + SetIncludeLicenses(scanCmd.includeLicenses). + SetPrintExtendedTable(scanCmd.printExtendedTable). + SetIsMultipleRootProject(true). + SetScanType(services.Binary). + PrintScanResults(); err != nil { + return + } + if err != nil { return err } diff --git a/xray/utils/analyzermanager.go b/xray/utils/analyzermanager.go index 12784921c..cf894d660 100644 --- a/xray/utils/analyzermanager.go +++ b/xray/utils/analyzermanager.go @@ -3,7 +3,6 @@ package utils import ( "errors" "fmt" - "github.com/jfrog/gofrog/version" "os" "os/exec" "path" @@ -24,7 +23,6 @@ const ( ApplicabilityFeatureId = "contextual_analysis" AnalyzerManagerZipName = "analyzerManager.zip" defaultAnalyzerManagerVersion = "1.3.2.2019257" - minAnalyzerManagerVersionForSast = "1.3" analyzerManagerDownloadPath = "xsc-gen-exe-analyzer-manager-local/v1" analyzerManagerDirName = "analyzerManager" analyzerManagerExecutableName = "analyzerManager" @@ -102,10 +100,21 @@ type AnalyzerManager struct { } func (am *AnalyzerManager) Exec(configFile, scanCommand, workingDir string, serverDetails *config.ServerDetails) (err error) { + return am.ExecWithOutputFile(configFile, scanCommand, workingDir, "", serverDetails) +} + +func (am *AnalyzerManager) ExecWithOutputFile(configFile, scanCommand, workingDir, outputFile string, serverDetails *config.ServerDetails) (err error) { if err = SetAnalyzerManagerEnvVariables(serverDetails); err != nil { return } - cmd := exec.Command(am.AnalyzerManagerFullPath, scanCommand, configFile, am.MultiScanId) + var cmd *exec.Cmd + if len(outputFile) > 0 { + log.Debug("Executing", am.AnalyzerManagerFullPath, scanCommand, configFile, outputFile, am.MultiScanId) + cmd = exec.Command(am.AnalyzerManagerFullPath, scanCommand, configFile, outputFile, am.MultiScanId) + } else { + log.Debug("Executing", am.AnalyzerManagerFullPath, scanCommand, configFile, am.MultiScanId) + cmd = exec.Command(am.AnalyzerManagerFullPath, scanCommand, configFile, am.MultiScanId) + } defer func() { if cmd.ProcessState != nil && !cmd.ProcessState.Exited() { if killProcessError := cmd.Process.Kill(); errorutils.CheckError(killProcessError) != nil { @@ -139,10 +148,6 @@ func GetAnalyzerManagerVersion() string { return defaultAnalyzerManagerVersion } -func IsSastSupported() bool { - return version.NewVersion(GetAnalyzerManagerVersion()).AtLeast(minAnalyzerManagerVersionForSast) -} - func GetAnalyzerManagerDirAbsolutePath() (string, error) { jfrogDir, err := config.GetJfrogDependenciesPath() if err != nil { diff --git a/xray/utils/analyzermanager_test.go b/xray/utils/analyzermanager_test.go index c8637a7e2..d493bfb4b 100644 --- a/xray/utils/analyzermanager_test.go +++ b/xray/utils/analyzermanager_test.go @@ -5,126 +5,9 @@ import ( "fmt" "testing" - "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" ) -func TestGetResultFileName(t *testing.T) { - fileNameValue := "fileNameValue" - tests := []struct { - result *sarif.Result - expectedOutput string - }{ - {result: &sarif.Result{ - Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{ArtifactLocation: &sarif.ArtifactLocation{URI: nil}}}, - }}, - expectedOutput: ""}, - {result: &sarif.Result{ - Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{ArtifactLocation: &sarif.ArtifactLocation{URI: &fileNameValue}}}, - }}, - expectedOutput: fileNameValue}, - } - - for _, test := range tests { - assert.Equal(t, test.expectedOutput, GetLocationFileName(test.result.Locations[0])) - } - -} - -func TestGetResultLocationInFile(t *testing.T) { - startLine := 19 - startColumn := 25 - - tests := []struct { - result *sarif.Result - expectedOutput string - }{ - {result: &sarif.Result{Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{Region: &sarif.Region{ - StartLine: &startLine, - StartColumn: &startColumn, - }}}}}, - expectedOutput: "19:25"}, - {result: &sarif.Result{Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{Region: &sarif.Region{ - StartLine: nil, - StartColumn: &startColumn, - }}}}}, - expectedOutput: ""}, - {result: &sarif.Result{Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{Region: &sarif.Region{ - StartLine: &startLine, - StartColumn: nil, - }}}}}, - expectedOutput: ""}, - {result: &sarif.Result{Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{Region: &sarif.Region{ - StartLine: nil, - StartColumn: nil, - }}}}}, - expectedOutput: ""}, - } - - for _, test := range tests { - assert.Equal(t, test.expectedOutput, GetStartLocationInFile(test.result.Locations[0])) - } -} - -func TestExtractRelativePath(t *testing.T) { - tests := []struct { - secretPath string - projectPath string - expectedResult string - }{ - {secretPath: "file:///Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", - projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "tests/req.nodejs/file.js"}, - {secretPath: "invalidSecretPath", - projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "invalidSecretPath"}, - {secretPath: "", - projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: ""}, - {secretPath: "file:///Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", - projectPath: "invalidProjectPath", expectedResult: "Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js"}, - {secretPath: "file:///private/Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", - projectPath: "invalidProjectPath", expectedResult: "Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js"}, - } - - for _, test := range tests { - assert.Equal(t, test.expectedResult, ExtractRelativePath(test.secretPath, test.projectPath)) - } -} - -func TestGetResultSeverity(t *testing.T) { - levelValueHigh := string(errorLevel) - levelValueMedium := string(warningLevel) - levelValueMedium2 := string(infoLevel) - levelValueLow := string(noteLevel) - levelValueUnknown := string(noneLevel) - - tests := []struct { - result *sarif.Result - expectedSeverity string - }{ - {result: &sarif.Result{}, - expectedSeverity: "Medium"}, - {result: &sarif.Result{Level: &levelValueHigh}, - expectedSeverity: "High"}, - {result: &sarif.Result{Level: &levelValueMedium}, - expectedSeverity: "Medium"}, - {result: &sarif.Result{Level: &levelValueMedium2}, - expectedSeverity: "Medium"}, - {result: &sarif.Result{Level: &levelValueLow}, - expectedSeverity: "Low"}, - {result: &sarif.Result{Level: &levelValueUnknown}, - expectedSeverity: "Unknown"}, - } - - for _, test := range tests { - assert.Equal(t, test.expectedSeverity, GetResultSeverity(test.result)) - } -} - func TestScanTypeErrorMsg(t *testing.T) { tests := []struct { scanner JasScanType diff --git a/xray/utils/auditbasicparams.go b/xray/utils/auditbasicparams.go index ff990e9fc..e5c739517 100644 --- a/xray/utils/auditbasicparams.go +++ b/xray/utils/auditbasicparams.go @@ -5,6 +5,32 @@ import ( ioUtils "github.com/jfrog/jfrog-client-go/utils/io" ) +type AuditParams interface { + DirectDependencies() []string + AppendDependenciesForApplicabilityScan(directDependencies []string) *AuditBasicParams + ServerDetails() (*config.ServerDetails, error) + SetServerDetails(serverDetails *config.ServerDetails) *AuditBasicParams + PipRequirementsFile() string + SetPipRequirementsFile(requirementsFile string) *AuditBasicParams + ExcludeTestDependencies() bool + SetExcludeTestDependencies(excludeTestDependencies bool) *AuditBasicParams + UseWrapper() bool + SetUseWrapper(useWrapper bool) *AuditBasicParams + InsecureTls() bool + SetInsecureTls(insecureTls bool) *AuditBasicParams + Technologies() []string + SetTechnologies(technologies []string) *AuditBasicParams + Progress() ioUtils.ProgressMgr + SetProgress(progress ioUtils.ProgressMgr) + Args() []string + SetNpmScope(depType string) *AuditBasicParams + OutputFormat() OutputFormat + DepsRepo() string + SetDepsRepo(depsRepo string) *AuditBasicParams + IgnoreConfigFile() bool + SetIgnoreConfigFile(ignoreConfigFile bool) *AuditBasicParams +} + type AuditBasicParams struct { serverDetails *config.ServerDetails outputFormat OutputFormat diff --git a/xray/utils/auditnpmparams.go b/xray/utils/auditnpmparams.go new file mode 100644 index 000000000..947c09c5a --- /dev/null +++ b/xray/utils/auditnpmparams.go @@ -0,0 +1,25 @@ +package utils + +type AuditNpmParams struct { + AuditParams + npmIgnoreNodeModules bool + npmOverwritePackageLock bool +} + +func (anp AuditNpmParams) SetNpmIgnoreNodeModules(ignoreNpmNodeModules bool) AuditNpmParams { + anp.npmIgnoreNodeModules = ignoreNpmNodeModules + return anp +} + +func (anp AuditNpmParams) SetNpmOverwritePackageLock(overwritePackageLock bool) AuditNpmParams { + anp.npmOverwritePackageLock = overwritePackageLock + return anp +} + +func (anp AuditNpmParams) NpmIgnoreNodeModules() bool { + return anp.npmIgnoreNodeModules +} + +func (anp AuditNpmParams) NpmOverwritePackageLock() bool { + return anp.npmOverwritePackageLock +} diff --git a/xray/utils/resultstable.go b/xray/utils/resultstable.go index ee04b677c..0b308b9ff 100644 --- a/xray/utils/resultstable.go +++ b/xray/utils/resultstable.go @@ -37,13 +37,13 @@ const ( // In case one (or more) of the violations contains the field FailBuild set to true, CliError with exit code 3 will be returned. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintViolationsTable(violations []services.Violation, extendedResults *ExtendedScanResults, multipleRoots, printExtended, isBinaryScan bool) error { +func PrintViolationsTable(violations []services.Violation, extendedResults *ExtendedScanResults, multipleRoots, printExtended bool, scanType services.ScanType) error { securityViolationsRows, licenseViolationsRows, operationalRiskViolationsRows, err := prepareViolations(violations, extendedResults, multipleRoots, true, true) if err != nil { return err } // Print tables, if scan is true; print the scan tables. - if isBinaryScan { + if scanType == services.Binary { err = coreutils.PrintTable(formats.ConvertToVulnerabilityScanTableRow(securityViolationsRows), "Security Violations", "No security violations were found", printExtended) if err != nil { return err @@ -182,13 +182,13 @@ func prepareViolations(violations []services.Violation, extendedResults *Extende // In case multipleRoots is true, the field Component will show the root of each impact path, otherwise it will show the root's child. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, extendedResults *ExtendedScanResults, multipleRoots, printExtended, isBinaryScan bool) error { +func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, extendedResults *ExtendedScanResults, multipleRoots, printExtended bool, scanType services.ScanType) error { vulnerabilitiesRows, err := prepareVulnerabilities(vulnerabilities, extendedResults, multipleRoots, true, true) if err != nil { return err } - if isBinaryScan { + if scanType == services.Binary { return coreutils.PrintTable(formats.ConvertToVulnerabilityScanTableRow(vulnerabilitiesRows), "Vulnerable Components", "✨ No vulnerable components were found ✨", printExtended) } var emptyTableMessage string @@ -266,12 +266,12 @@ func sortVulnerabilityOrViolationRows(rows []formats.VulnerabilityOrViolationRow // In case multipleRoots is true, the field Component will show the root of each impact path, otherwise it will show the root's child. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintLicensesTable(licenses []services.License, printExtended, isBinaryScan bool) error { +func PrintLicensesTable(licenses []services.License, printExtended bool, scanType services.ScanType) error { licensesRows, err := PrepareLicenses(licenses) if err != nil { return err } - if isBinaryScan { + if scanType == services.Binary { return coreutils.PrintTable(formats.ConvertToLicenseScanTableRow(licensesRows), "Licenses", "No licenses were found", printExtended) } return coreutils.PrintTable(formats.ConvertToLicenseTableRow(licensesRows), "Licenses", "No licenses were found", printExtended) @@ -420,6 +420,7 @@ func prepareSast(sasts []*sarif.Run, isTable bool) []formats.SourceCodeRow { formats.SourceCodeRow{ SeverityDetails: formats.SeverityDetails{Severity: currSeverity.printableTitle(isTable), SeverityNumValue: currSeverity.NumValue()}, ScannerDescription: scannerDescription, + Finding: GetResultMsgText(sastResult), Location: formats.Location{ File: GetRelativeLocationFileName(location, sastRun.Invocations), StartLine: GetLocationStartLine(location), diff --git a/xray/utils/resultstable_test.go b/xray/utils/resultstable_test.go index 66d72bfb3..d0589fe24 100644 --- a/xray/utils/resultstable_test.go +++ b/xray/utils/resultstable_test.go @@ -25,7 +25,7 @@ func TestPrintViolationsTable(t *testing.T) { } for _, test := range tests { - err := PrintViolationsTable(test.violations, &ExtendedScanResults{}, false, true, true) + err := PrintViolationsTable(test.violations, &ExtendedScanResults{}, false, true, services.Binary) assert.NoError(t, err) if CheckIfFailBuild([]services.ScanResponse{{Violations: test.violations}}) { err = NewFailBuildError() @@ -439,9 +439,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyResultWithOneLocation("fileName1", 0, 1, "snippet1", "applic_testCve1", "info"), - getDummyPassingResult("applic_testCve2"), + CreateRunWithDummyResults( + CreateResultWithOneLocation("fileName1", 0, 1, 0, 0, "snippet1", "applic_testCve1", "info"), + CreateDummyPassingResult("applic_testCve2"), ), }, EntitledForJas: true, @@ -453,9 +453,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyPassingResult("applic_testCve1"), - getDummyResultWithOneLocation("fileName2", 1, 0, "snippet2", "applic_testCve2", "warning"), + CreateRunWithDummyResults( + CreateDummyPassingResult("applic_testCve1"), + CreateResultWithOneLocation("fileName2", 1, 0, 0, 0, "snippet2", "applic_testCve2", "warning"), ), }, EntitledForJas: true, @@ -467,9 +467,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyPassingResult("applic_testCve1"), - getDummyResultWithOneLocation("fileName3", 0, 1, "snippet3", "applic_testCve2", "info"), + CreateRunWithDummyResults( + CreateDummyPassingResult("applic_testCve1"), + CreateResultWithOneLocation("fileName3", 0, 1, 0, 0, "snippet3", "applic_testCve2", "info"), ), }, EntitledForJas: true, @@ -481,9 +481,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyPassingResult("applic_testCve1"), - getDummyPassingResult("applic_testCve2"), + CreateRunWithDummyResults( + CreateDummyPassingResult("applic_testCve1"), + CreateDummyPassingResult("applic_testCve2"), ), }, EntitledForJas: true, @@ -495,9 +495,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyPassingResult("applic_testCve1"), - getDummyResultWithOneLocation("fileName4", 1, 0, "snippet", "applic_testCve2", "warning"), + CreateRunWithDummyResults( + CreateDummyPassingResult("applic_testCve1"), + CreateResultWithOneLocation("fileName4", 1, 0, 0, 0, "snippet", "applic_testCve2", "warning"), ), }, EntitledForJas: true, @@ -509,7 +509,7 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults(getDummyPassingResult("applic_testCve1")), + CreateRunWithDummyResults(CreateDummyPassingResult("applic_testCve1")), }, EntitledForJas: true}, cves: []services.Cve{{Id: "testCve1"}, {Id: "testCve2"}}, @@ -706,6 +706,344 @@ func TestShouldDisqualifyEvidence(t *testing.T) { } } +func TestPrepareIac(t *testing.T) { + testCases := []struct { + name string + input []*sarif.Run + expectedOutput []formats.SourceCodeRow + }{ + { + name: "No Iac run", + input: []*sarif.Run{}, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Iac run - no results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + }, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Iac run - with results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults( + CreateResultWithLocations("iac finding", "rule1", "info", + CreateLocation("file://wd/file", 1, 2, 3, 4, "snippet"), + CreateLocation("file://wd/file2", 5, 6, 7, 8, "other-snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults( + CreateResultWithLocations("other iac finding", "rule2", "error", + CreateLocation("file://wd2/file3", 1, 2, 3, 4, "snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + expectedOutput: []formats.SourceCodeRow{ + { + SeverityDetails: formats.SeverityDetails{ + Severity: "High", + SeverityNumValue: 13, + }, + Finding: "other iac finding", + Location: formats.Location{ + File: "file3", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "iac finding", + Location: formats.Location{ + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "iac finding", + Location: formats.Location{ + File: "file2", + StartLine: 5, + StartColumn: 6, + EndLine: 7, + EndColumn: 8, + Snippet: "other-snippet", + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.ElementsMatch(t, tc.expectedOutput, prepareIacs(tc.input, false)) + }) + } +} + +func TestPrepareSecrets(t *testing.T) { + testCases := []struct { + name string + input []*sarif.Run + expectedOutput []formats.SourceCodeRow + }{ + { + name: "No Secret run", + input: []*sarif.Run{}, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Secret run - no results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + }, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Secret run - with results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults( + CreateResultWithLocations("secret finding", "rule1", "info", + CreateLocation("file://wd/file", 1, 2, 3, 4, "some-secret-snippet"), + CreateLocation("file://wd/file2", 5, 6, 7, 8, "other-secret-snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults( + CreateResultWithLocations("other secret finding", "rule2", "note", + CreateLocation("file://wd2/file3", 1, 2, 3, 4, "some-secret-snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + expectedOutput: []formats.SourceCodeRow{ + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Low", + SeverityNumValue: 9, + }, + Finding: "other secret finding", + Location: formats.Location{ + File: "file3", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "some-secret-snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "secret finding", + Location: formats.Location{ + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "some-secret-snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "secret finding", + Location: formats.Location{ + File: "file2", + StartLine: 5, + StartColumn: 6, + EndLine: 7, + EndColumn: 8, + Snippet: "other-secret-snippet", + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.ElementsMatch(t, tc.expectedOutput, prepareSecrets(tc.input, false)) + }) + } +} + +func TestPrepareSast(t *testing.T) { + testCases := []struct { + name string + input []*sarif.Run + expectedOutput []formats.SourceCodeRow + }{ + { + name: "No Sast run", + input: []*sarif.Run{}, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Sast run - no results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + }, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Sast run - with results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults( + CreateResultWithLocations("sast finding", "rule1", "info", + CreateLocation("file://wd/file", 1, 2, 3, 4, "snippet"), + CreateLocation("file://wd/file2", 5, 6, 7, 8, "other-snippet"), + ).WithCodeFlows([]*sarif.CodeFlow{ + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file://wd/file2", 0, 2, 0, 2, "snippetA"), + CreateLocation("file://wd/file", 1, 2, 3, 4, "snippet"), + )), + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file://wd/file4", 1, 0, 1, 8, "snippetB"), + CreateLocation("file://wd/file", 1, 2, 3, 4, "snippet"), + )), + }), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults( + CreateResultWithLocations("other sast finding", "rule2", "error", + CreateLocation("file://wd2/file3", 1, 2, 3, 4, "snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + expectedOutput: []formats.SourceCodeRow{ + { + SeverityDetails: formats.SeverityDetails{ + Severity: "High", + SeverityNumValue: 13, + }, + Finding: "other sast finding", + Location: formats.Location{ + File: "file3", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "sast finding", + Location: formats.Location{ + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + CodeFlow: [][]formats.Location{ + { + { + File: "file2", + StartLine: 0, + StartColumn: 2, + EndLine: 0, + EndColumn: 2, + Snippet: "snippetA", + }, + { + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + { + { + File: "file4", + StartLine: 1, + StartColumn: 0, + EndLine: 1, + EndColumn: 8, + Snippet: "snippetB", + }, + { + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "sast finding", + Location: formats.Location{ + File: "file2", + StartLine: 5, + StartColumn: 6, + EndLine: 7, + EndColumn: 8, + Snippet: "other-snippet", + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.ElementsMatch(t, tc.expectedOutput, prepareSast(tc.input, false)) + }) + } +} + func newBoolPtr(v bool) *bool { return &v } diff --git a/xray/utils/resultwriter.go b/xray/utils/resultwriter.go index 1e8abcca4..553f37388 100644 --- a/xray/utils/resultwriter.go +++ b/xray/utils/resultwriter.go @@ -25,6 +25,8 @@ const ( Json OutputFormat = "json" SimpleJson OutputFormat = "simple-json" Sarif OutputFormat = "sarif" + + BaseDocumentationURL = "https://docs.jfrog-applications.jfrog.io/jfrog-security-features/" ) const MissingCveScore = "0" @@ -34,32 +36,92 @@ var OutputFormats = []string{string(Table), string(Json), string(SimpleJson), st var CurationOutputFormats = []string{string(Table), string(Json)} +type ResultsWriter struct { + // The scan results. + results *ExtendedScanResults + // SimpleJsonError Errors to be added to output of the SimpleJson format. + simpleJsonError []formats.SimpleJsonError + // Format The output format. + format OutputFormat + // IncludeVulnerabilities If true, include all vulnerabilities as part of the output. Else, include violations only. + includeVulnerabilities bool + // IncludeLicenses If true, also include license violations as part of the output. + includeLicenses bool + // IsMultipleRoots multipleRoots is set to true, in case the given results array contains (or may contain) results of several projects (like in binary scan). + isMultipleRoots bool + // PrintExtended, If true, show extended results. + printExtended bool + // The scanType (binary,dependency) + scanType services.ScanType + // Messages - Option array of messages, to be displayed if the format is Table + messages []string +} + +func NewResultsWriter(extendedScanResults *ExtendedScanResults) *ResultsWriter { + return &ResultsWriter{results: extendedScanResults} +} + +func (rw *ResultsWriter) SetOutputFormat(format OutputFormat) *ResultsWriter { + rw.format = format + return rw +} + +func (rw *ResultsWriter) SetScanType(scanType services.ScanType) *ResultsWriter { + rw.scanType = scanType + return rw +} + +func (rw *ResultsWriter) SetSimpleJsonError(jsonErrors []formats.SimpleJsonError) *ResultsWriter { + rw.simpleJsonError = jsonErrors + return rw +} + +func (rw *ResultsWriter) SetIncludeVulnerabilities(includeVulnerabilities bool) *ResultsWriter { + rw.includeVulnerabilities = includeVulnerabilities + return rw +} + +func (rw *ResultsWriter) SetIncludeLicenses(licenses bool) *ResultsWriter { + rw.includeLicenses = licenses + return rw +} + +func (rw *ResultsWriter) SetIsMultipleRootProject(isMultipleRootProject bool) *ResultsWriter { + rw.isMultipleRoots = isMultipleRootProject + return rw +} + +func (rw *ResultsWriter) SetPrintExtendedTable(extendedTable bool) *ResultsWriter { + rw.printExtended = extendedTable + return rw +} + +func (rw *ResultsWriter) SetExtraMessages(messages []string) *ResultsWriter { + rw.messages = messages + return rw + +} + // PrintScanResults prints the scan results in the specified format. // Note that errors are printed only with SimpleJson format. -// -// results - The scan results. -// simpleJsonError - Errors to be added to output of the SimpleJson format. -// format - The output format. -// includeVulnerabilities - If trie, include all vulnerabilities as part of the output. Else, include violations only. -// includeLicenses - If true, also include license violations as part of the output. -// isMultipleRoots - multipleRoots is set to true, in case the given results array contains (or may contain) results of several projects (like in binary scan). -// printExtended -If true, show extended results. -// scan - If true, use an output layout suitable for `jf scan` or `jf docker scan` results. Otherwise, use a layout compatible for `jf audit` . -// messages - Option array of messages, to be displayed if the format is Table -func PrintScanResults(results *ExtendedScanResults, simpleJsonError []formats.SimpleJsonError, format OutputFormat, includeVulnerabilities, includeLicenses, isMultipleRoots, printExtended, isBinaryScan bool, messages []string) error { - switch format { +func (rw *ResultsWriter) PrintScanResults() error { + switch rw.format { case Table: - return printScanResultsTables(results, isBinaryScan, includeVulnerabilities, includeLicenses, isMultipleRoots, printExtended, messages) + return rw.printScanResultsTables() case SimpleJson: - jsonTable, err := convertScanToSimpleJson(results, simpleJsonError, isMultipleRoots, includeLicenses, false) + jsonTable, err := rw.convertScanToSimpleJson() if err != nil { return err } return PrintJson(jsonTable) case Json: - return PrintJson(results.getXrayScanResults()) + return PrintJson(rw.results.getXrayScanResults()) case Sarif: - sarifFile, err := GenerateSarifContentFromResults(results, isMultipleRoots, includeLicenses, false) + sarifReport, err := GenereateSarifReportFromResults(rw.results, rw.isMultipleRoots, rw.includeLicenses) + if err != nil { + return err + } + sarifFile, err := ConvertSarifReportToString(sarifReport) if err != nil { return err } @@ -67,41 +129,37 @@ func PrintScanResults(results *ExtendedScanResults, simpleJsonError []formats.Si } return nil } - -func printScanResultsTables(results *ExtendedScanResults, isBinaryScan, includeVulnerabilities, includeLicenses, isMultipleRoots, printExtended bool, messages []string) (err error) { - printMessages(messages) - violations, vulnerabilities, licenses := SplitScanResults(results.getXrayScanResults()) - if len(results.getXrayScanResults()) > 0 { +func (rw *ResultsWriter) printScanResultsTables() (err error) { + printMessages(rw.messages) + violations, vulnerabilities, licenses := SplitScanResults(rw.results.getXrayScanResults()) + if len(rw.results.getXrayScanResults()) > 0 { var resultsPath string - if resultsPath, err = writeJsonResults(results); err != nil { + if resultsPath, err = writeJsonResults(rw.results); err != nil { return } printMessage(coreutils.PrintTitle("The full scan results are available here: ") + coreutils.PrintLink(resultsPath)) } log.Output() - if includeVulnerabilities { - err = PrintVulnerabilitiesTable(vulnerabilities, results, isMultipleRoots, printExtended, isBinaryScan) + if rw.includeVulnerabilities { + err = PrintVulnerabilitiesTable(vulnerabilities, rw.results, rw.isMultipleRoots, rw.printExtended, rw.scanType) } else { - err = PrintViolationsTable(violations, results, isMultipleRoots, printExtended, isBinaryScan) + err = PrintViolationsTable(violations, rw.results, rw.isMultipleRoots, rw.printExtended, rw.scanType) } if err != nil { return } - if includeLicenses { - if err = PrintLicensesTable(licenses, printExtended, isBinaryScan); err != nil { + if rw.includeLicenses { + if err = PrintLicensesTable(licenses, rw.printExtended, rw.scanType); err != nil { return } } - if err = PrintSecretsTable(results.SecretsScanResults, results.EntitledForJas); err != nil { + if err = PrintSecretsTable(rw.results.SecretsScanResults, rw.results.EntitledForJas); err != nil { return } - if err = PrintIacTable(results.IacScanResults, results.EntitledForJas); err != nil { + if err = PrintIacTable(rw.results.IacScanResults, rw.results.EntitledForJas); err != nil { return } - if !IsSastSupported() { - return - } - return PrintSastTable(results.SastScanResults, results.EntitledForJas) + return PrintSastTable(rw.results.SastScanResults, rw.results.EntitledForJas) } func printMessages(messages []string) { @@ -117,12 +175,12 @@ func printMessage(message string) { log.Output("💬" + message) } -func GenerateSarifContentFromResults(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, markdownOutput bool) (sarifStr string, err error) { - report, err := NewReport() +func GenereateSarifReportFromResults(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses bool) (report *sarif.Report, err error) { + report, err = NewReport() if err != nil { return } - xrayRun, err := convertXrayResponsesToSarifRun(extendedResults, isMultipleRoots, includeLicenses, markdownOutput) + xrayRun, err := convertXrayResponsesToSarifRun(extendedResults, isMultipleRoots, includeLicenses) if err != nil { return } @@ -133,23 +191,26 @@ func GenerateSarifContentFromResults(extendedResults *ExtendedScanResults, isMul report.Runs = append(report.Runs, extendedResults.SecretsScanResults...) report.Runs = append(report.Runs, extendedResults.SastScanResults...) + return +} + +func ConvertSarifReportToString(report *sarif.Report) (sarifStr string, err error) { out, err := json.Marshal(report) if err != nil { return "", errorutils.CheckError(err) } - return clientUtils.IndentJson(out), nil } -func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, markdownOutput bool) (run *sarif.Run, err error) { +func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses bool) (run *sarif.Run, err error) { xrayJson, err := convertXrayScanToSimpleJson(extendedResults, isMultipleRoots, includeLicenses, true) if err != nil { return } - xrayRun := sarif.NewRunWithInformationURI("JFrog Xray Sca", "https://jfrog.com/xray/") + xrayRun := sarif.NewRunWithInformationURI("JFrog Xray SCA", BaseDocumentationURL+"sca") xrayRun.Tool.Driver.Version = &extendedResults.XrayVersion if len(xrayJson.Vulnerabilities) > 0 || len(xrayJson.SecurityViolations) > 0 { - if err = extractXrayIssuesToSarifRun(xrayRun, xrayJson, markdownOutput); err != nil { + if err = extractXrayIssuesToSarifRun(xrayRun, xrayJson); err != nil { return } } @@ -157,20 +218,19 @@ func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMult return } -func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResults, markdownOutput bool) error { +func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResults) error { for _, vulnerability := range xrayJson.Vulnerabilities { if err := addXrayCveIssueToSarifRun( vulnerability.Cves, vulnerability.IssueId, vulnerability.Severity, - vulnerability.Technology.GetPackageDescriptor(), + vulnerability.Technology, vulnerability.Components, vulnerability.Applicable, vulnerability.ImpactedDependencyName, vulnerability.ImpactedDependencyVersion, vulnerability.Summary, vulnerability.FixedVersions, - markdownOutput, run, ); err != nil { return err @@ -181,14 +241,13 @@ func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResu violation.Cves, violation.IssueId, violation.Severity, - violation.Technology.GetPackageDescriptor(), + violation.Technology, violation.Components, violation.Applicable, violation.ImpactedDependencyName, violation.ImpactedDependencyVersion, violation.Summary, violation.FixedVersions, - markdownOutput, run, ); err != nil { return err @@ -203,35 +262,66 @@ func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResu return nil } -func addXrayCveIssueToSarifRun(cves []formats.CveRow, issueId, severity, file string, components []formats.ComponentRow, applicable, impactedDependencyName, impactedDependencyVersion, summary string, fixedVersions []string, markdownOutput bool, run *sarif.Run) error { +func addXrayCveIssueToSarifRun(cves []formats.CveRow, issueId, severity string, tech coreutils.Technology, components []formats.ComponentRow, applicable, impactedDependencyName, impactedDependencyVersion, summary string, fixedVersions []string, run *sarif.Run) error { maxCveScore, err := findMaxCVEScore(cves) if err != nil { return err } cveId := GetIssueIdentifier(cves, issueId) msg := getVulnerabilityOrViolationSarifHeadline(impactedDependencyName, impactedDependencyVersion, cveId) - location := sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri(file))) - + location, err := getXrayIssueLocationIfValidExists(tech, run) + if err != nil { + return err + } if rule, isNewRule := addResultToSarifRun(cveId, msg, severity, location, run); isNewRule { cveRuleProperties := sarif.NewPropertyBag() if maxCveScore != MissingCveScore { cveRuleProperties.Add("security-severity", maxCveScore) } rule.WithProperties(cveRuleProperties.Properties) - if markdownOutput { - formattedDirectDependencies, err := getDirectDependenciesFormatted(components) - if err != nil { - return err - } - markdownDescription := getSarifTableDescription(formattedDirectDependencies, maxCveScore, applicable, fixedVersions) + "\n" - rule.WithMarkdownHelp(markdownDescription) - } else { - rule.WithDescription(summary) + formattedDirectDependencies, err := getDirectDependenciesFormatted(components) + if err != nil { + return err } + markdownDescription := getSarifTableDescription(formattedDirectDependencies, maxCveScore, applicable, fixedVersions) + "\n" + rule.WithHelp(&sarif.MultiformatMessageString{ + Text: &summary, + Markdown: &markdownDescription, + }) } return nil } +func getDescriptorFullPath(tech coreutils.Technology, run *sarif.Run) (string, error) { + descriptors := tech.GetPackageDescriptor() + if len(descriptors) == 1 { + // Generate the full path + return GetFullLocationFileName(strings.TrimSpace(descriptors[0]), run.Invocations), nil + } + for _, descriptor := range descriptors { + // If multiple options return first to match + absolutePath := GetFullLocationFileName(strings.TrimSpace(descriptor), run.Invocations) + if exists, err := fileutils.IsFileExists(absolutePath, false); err != nil { + return "", err + } else if exists { + return absolutePath, nil + } + } + return "", nil +} + +// Get the descriptor location with the Xray issues if exists. +func getXrayIssueLocationIfValidExists(tech coreutils.Technology, run *sarif.Run) (location *sarif.Location, err error) { + descriptorPath, err := getDescriptorFullPath(tech, run) + if err != nil { + return + } + if strings.TrimSpace(descriptorPath) == "" { + return + } + return sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri("file://" + descriptorPath))), nil +} + func addResultToSarifRun(issueId, msg, severity string, location *sarif.Location, run *sarif.Run) (rule *sarif.ReportingDescriptor, isNewRule bool) { if rule, _ = run.GetRuleById(issueId); rule == nil { isNewRule = true @@ -273,21 +363,21 @@ func convertXrayScanToSimpleJson(extendedResults *ExtendedScanResults, isMultipl return jsonTable, nil } -func convertScanToSimpleJson(extendedResults *ExtendedScanResults, errors []formats.SimpleJsonError, isMultipleRoots, includeLicenses, simplifiedOutput bool) (formats.SimpleJsonResults, error) { - jsonTable, err := convertXrayScanToSimpleJson(extendedResults, isMultipleRoots, includeLicenses, simplifiedOutput) +func (rw *ResultsWriter) convertScanToSimpleJson() (formats.SimpleJsonResults, error) { + jsonTable, err := convertXrayScanToSimpleJson(rw.results, rw.isMultipleRoots, rw.includeLicenses, false) if err != nil { return formats.SimpleJsonResults{}, err } - if len(extendedResults.SecretsScanResults) > 0 { - jsonTable.Secrets = PrepareSecrets(extendedResults.SecretsScanResults) + if len(rw.results.SecretsScanResults) > 0 { + jsonTable.Secrets = PrepareSecrets(rw.results.SecretsScanResults) } - if len(extendedResults.IacScanResults) > 0 { - jsonTable.Iacs = PrepareIacs(extendedResults.IacScanResults) + if len(rw.results.IacScanResults) > 0 { + jsonTable.Iacs = PrepareIacs(rw.results.IacScanResults) } - if len(extendedResults.SastScanResults) > 0 { - jsonTable.Sast = PrepareSast(extendedResults.SastScanResults) + if len(rw.results.SastScanResults) > 0 { + jsonTable.Sast = PrepareSast(rw.results.SastScanResults) } - jsonTable.Errors = errors + jsonTable.Errors = rw.simpleJsonError return jsonTable, nil } diff --git a/xray/utils/resultwriter_test.go b/xray/utils/resultwriter_test.go index 57458ea34..a67dfe86c 100644 --- a/xray/utils/resultwriter_test.go +++ b/xray/utils/resultwriter_test.go @@ -1,9 +1,14 @@ package utils import ( + "os" + "path/filepath" "testing" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-cli-core/v2/utils/tests" "github.com/jfrog/jfrog-cli-core/v2/xray/formats" + "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" ) @@ -134,3 +139,57 @@ func TestFindMaxCVEScore(t *testing.T) { }) } } + +func TestGetXrayIssueLocationIfValidExists(t *testing.T) { + testDir, cleanup := tests.CreateTempDirWithCallbackAndAssert(t) + defer cleanup() + invocation := sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation(testDir)) + file, err := os.Create(filepath.Join(testDir, "go.mod")) + assert.NoError(t, err) + assert.NotNil(t, file) + defer func() { assert.NoError(t, file.Close()) }() + file2, err := os.Create(filepath.Join(testDir, "build.gradle.kts")) + assert.NoError(t, err) + assert.NotNil(t, file2) + defer func() { assert.NoError(t, file2.Close()) }() + + testCases := []struct { + name string + tech coreutils.Technology + run *sarif.Run + expectedOutput *sarif.Location + }{ + { + name: "No descriptor information", + tech: coreutils.Pip, + run: CreateRunWithDummyResults().WithInvocations([]*sarif.Invocation{invocation}), + expectedOutput: nil, + }, + { + name: "One descriptor information", + tech: coreutils.Go, + run: CreateRunWithDummyResults().WithInvocations([]*sarif.Invocation{invocation}), + expectedOutput: sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri("file://" + filepath.Join(testDir, "go.mod")))), + }, + { + name: "One descriptor information - no invocation", + tech: coreutils.Go, + run: CreateRunWithDummyResults(), + expectedOutput: sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri("file://go.mod"))), + }, + { + name: "Multiple descriptor information", + tech: coreutils.Gradle, + run: CreateRunWithDummyResults().WithInvocations([]*sarif.Invocation{invocation}), + expectedOutput: sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri("file://" + filepath.Join(testDir, "build.gradle.kts")))), + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + output, err := getXrayIssueLocationIfValidExists(tc.tech, tc.run) + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedOutput, output) + } + }) + } +} diff --git a/xray/utils/sarifutils.go b/xray/utils/sarifutils.go index 5b5d31ab2..0da577140 100644 --- a/xray/utils/sarifutils.go +++ b/xray/utils/sarifutils.go @@ -3,7 +3,6 @@ package utils import ( "fmt" "path/filepath" - "strconv" "strings" "github.com/jfrog/jfrog-client-go/utils/errorutils" @@ -115,24 +114,6 @@ func GetResultsLocationCount(runs ...*sarif.Run) (count int) { return } -func GetLevelResultsLocationCount(run *sarif.Run, level SarifLevel) (count int) { - for _, result := range run.Results { - if level == SarifLevel(*result.Level) { - count += len(result.Locations) - } - } - return -} - -func GetResultsByRuleId(run *sarif.Run, ruleId string) (results []*sarif.Result) { - for _, result := range run.Results { - if *result.RuleID == ruleId { - results = append(results, result) - } - } - return -} - func GetResultMsgText(result *sarif.Result) string { if result.Message.Text != nil { return *result.Message.Text @@ -141,19 +122,11 @@ func GetResultMsgText(result *sarif.Result) string { } func GetLocationSnippet(location *sarif.Location) string { - snippet := GetLocationSnippetPointer(location) - if snippet == nil { - return "" - } - return *snippet -} - -func GetLocationSnippetPointer(location *sarif.Location) *string { region := getLocationRegion(location) if region != nil && region.Snippet != nil { - return region.Snippet.Text + return *region.Snippet.Text } - return nil + return "" } func SetLocationSnippet(location *sarif.Location, snippet string) { @@ -163,9 +136,8 @@ func SetLocationSnippet(location *sarif.Location, snippet string) { } func GetLocationFileName(location *sarif.Location) string { - filePath := location.PhysicalLocation.ArtifactLocation.URI - if filePath != nil { - return *filePath + if location != nil && location.PhysicalLocation != nil && location.PhysicalLocation.ArtifactLocation != nil && location.PhysicalLocation.ArtifactLocation.URI != nil { + return *location.PhysicalLocation.ArtifactLocation.URI } return "" } @@ -175,7 +147,6 @@ func GetRelativeLocationFileName(location *sarif.Location, invocations []*sarif. if len(invocations) > 0 { wd = GetInvocationWorkingDirectory(invocations[0]) } - GetLocationFileName(location) filePath := GetLocationFileName(location) if filePath != "" { return ExtractRelativePath(filePath, wd) @@ -183,8 +154,15 @@ func GetRelativeLocationFileName(location *sarif.Location, invocations []*sarif. return "" } +func GetFullLocationFileName(relative string, invocations []*sarif.Invocation) string { + if len(invocations) == 0 { + return relative + } + return filepath.Join(GetInvocationWorkingDirectory(invocations[0]), relative) +} + func SetLocationFileName(location *sarif.Location, fileName string) { - if location != nil && location.PhysicalLocation != nil && location.PhysicalLocation.Region != nil && location.PhysicalLocation.Region.Snippet != nil { + if location != nil && location.PhysicalLocation != nil && location.PhysicalLocation.ArtifactLocation != nil { location.PhysicalLocation.ArtifactLocation.URI = &fileName } } @@ -228,15 +206,6 @@ func GetLocationEndColumn(location *sarif.Location) int { return 0 } -func GetStartLocationInFile(location *sarif.Location) string { - startLine := location.PhysicalLocation.Region.StartLine - startColumn := location.PhysicalLocation.Region.StartColumn - if startLine != nil && startColumn != nil { - return strconv.Itoa(*startLine) + ":" + strconv.Itoa(*startColumn) - } - return "" -} - func ExtractRelativePath(resultPath string, projectRoot string) string { // Remove OS-specific file prefix resultPath = strings.TrimPrefix(resultPath, "file:///private") @@ -291,7 +260,7 @@ func GetRunRules(run *sarif.Run) []*sarif.ReportingDescriptor { } func GetInvocationWorkingDirectory(invocation *sarif.Invocation) string { - if invocation.WorkingDirectory != nil && invocation.WorkingDirectory.URI != nil { + if invocation != nil && invocation.WorkingDirectory != nil && invocation.WorkingDirectory.URI != nil { return *invocation.WorkingDirectory.URI } return "" diff --git a/xray/utils/sarifutils_test.go b/xray/utils/sarifutils_test.go index 4e0031268..33358fe7a 100644 --- a/xray/utils/sarifutils_test.go +++ b/xray/utils/sarifutils_test.go @@ -1,43 +1,672 @@ package utils import ( - "github.com/jfrog/gofrog/datastructures" + "path/filepath" + "testing" + "github.com/owenrumney/go-sarif/v2/sarif" + "github.com/stretchr/testify/assert" ) -func getRunWithDummyResults(results ...*sarif.Result) *sarif.Run { - run := sarif.NewRunWithInformationURI("", "") - ids := datastructures.MakeSet[string]() - for _, result := range results { - if !ids.Exists(*result.RuleID) { - run.Tool.Driver.Rules = append(run.Tool.Driver.Rules, sarif.NewRule(*result.RuleID)) - ids.Add(*result.RuleID) - } +func TestAggregateMultipleRunsIntoSingle(t *testing.T) { + tests := []struct { + runs []*sarif.Run + expectedOutput *sarif.Run + }{ + { + runs: []*sarif.Run{}, + expectedOutput: CreateRunWithDummyResults(), + }, + { + runs: []*sarif.Run{ + CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule2", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults(), + }, + expectedOutput: CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule2", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + }, + { + runs: []*sarif.Run{ + CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule2", "level"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule3", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults( + CreateResultWithLocations("msg", "rule2", "level", + CreateLocation("file", 1, 2, 3, 4, "snippet"), + CreateLocation("file2", 1, 2, 3, 4, "other-snippet"), + ), + CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet2", "rule2", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + expectedOutput: CreateRunWithDummyResults( + // First run results + CreateDummyPassingResult("rule1"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule2", "level"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule3", "level"), + // Second run results + CreateResultWithLocations("msg", "rule2", "level", + CreateLocation("file", 1, 2, 3, 4, "snippet"), + CreateLocation("file2", 1, 2, 3, 4, "other-snippet"), + ), + CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet2", "rule2", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + } + + for _, test := range tests { + run := CreateRunWithDummyResults() + AggregateMultipleRunsIntoSingle(test.runs, run) + assert.Equal(t, test.expectedOutput, run) + } +} + +func TestGetLocationRelatedCodeFlowsFromResult(t *testing.T) { + tests := []struct { + result *sarif.Result + location *sarif.Location + expectedOutput []*sarif.CodeFlow + }{ + { + result: CreateDummyPassingResult("rule"), + location: CreateLocation("file", 0, 0, 0, 0, "snippet"), + expectedOutput: nil, + }, + { + result: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level"), + location: CreateLocation("file", 0, 0, 0, 0, "snippet"), + expectedOutput: nil, + }, + { + result: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level").WithCodeFlows([]*sarif.CodeFlow{CreateCodeFlow(CreateThreadFlow(CreateLocation("file", 0, 0, 0, 0, "snippet")))}), + location: CreateLocation("file2", 0, 0, 0, 0, "snippet"), + expectedOutput: nil, + }, + { + result: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level").WithCodeFlows([]*sarif.CodeFlow{CreateCodeFlow(CreateThreadFlow(CreateLocation("file", 0, 0, 0, 0, "snippet")))}), + location: CreateLocation("file", 0, 0, 0, 0, "snippet"), + expectedOutput: []*sarif.CodeFlow{CreateCodeFlow(CreateThreadFlow(CreateLocation("file", 0, 0, 0, 0, "snippet")))}, + }, + { + result: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level").WithCodeFlows([]*sarif.CodeFlow{ + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file4", 2, 0, 2, 0, "snippetB"), + CreateLocation("file2", 0, 2, 0, 2, "snippetA"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + )), + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file", 0, 0, 0, 0, "snippet"), + CreateLocation("file2", 1, 0, 1, 0, "snippet"), + )), + CreateCodeFlow(CreateThreadFlow( + CreateLocation("fileC", 1, 1, 1, 1, "snippetC"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + )), + }), + location: CreateLocation("file", 0, 0, 0, 0, "snippet"), + expectedOutput: []*sarif.CodeFlow{ + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file4", 2, 0, 2, 0, "snippetB"), + CreateLocation("file2", 0, 2, 0, 2, "snippetA"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + )), + CreateCodeFlow(CreateThreadFlow( + CreateLocation("fileC", 1, 1, 1, 1, "snippetC"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + )), + }, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationRelatedCodeFlowsFromResult(test.location, test.result)) + } +} + +func TestGetResultsLocationCount(t *testing.T) { + tests := []struct { + runs []*sarif.Run + expectedOutput int + }{ + { + runs: []*sarif.Run{}, + expectedOutput: 0, + }, + { + runs: []*sarif.Run{CreateRunWithDummyResults()}, + expectedOutput: 0, + }, + { + runs: []*sarif.Run{CreateRunWithDummyResults( + CreateDummyPassingResult("rule"), + CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level"), + )}, + expectedOutput: 1, + }, + { + runs: []*sarif.Run{ + CreateRunWithDummyResults( + CreateDummyPassingResult("rule"), + CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level"), + ), + CreateRunWithDummyResults( + CreateResultWithLocations( + "msg", + "rule", + "level", + CreateLocation("file", 0, 0, 0, 0, "snippet"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + ), + ), + }, + expectedOutput: 4, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetResultsLocationCount(test.runs...)) + } +} + +func TestGetResultMsgText(t *testing.T) { + tests := []struct { + result *sarif.Result + expectedOutput string + }{ + { + result: &sarif.Result{}, + expectedOutput: "", + }, + { + result: CreateResultWithLocations("msg", "rule", "level"), + expectedOutput: "msg", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetResultMsgText(test.result)) + } +} + +func TestGetLocationSnippet(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput string + }{ + { + location: nil, + expectedOutput: "", + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: "snippet", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationSnippet(test.location)) } - return run.WithResults(results) } -func getDummyPassingResult(ruleId string) *sarif.Result { - kind := "pass" - return &sarif.Result{ - Kind: &kind, - RuleID: &ruleId, +func TestSetLocationSnippet(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput string + }{ + { + location: nil, + expectedOutput: "", + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: "changedSnippet", + }, + } + + for _, test := range tests { + SetLocationSnippet(test.location, test.expectedOutput) + assert.Equal(t, test.expectedOutput, GetLocationSnippet(test.location)) + } +} + +func TestGetLocationFileName(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput string + }{ + { + location: nil, + expectedOutput: "", + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: "filename", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationFileName(test.location)) } } -func getDummyResultWithOneLocation(fileName string, startLine, startCol int, snippet, ruleId string, level string) *sarif.Result { - return &sarif.Result{ - Locations: []*sarif.Location{ - { - PhysicalLocation: &sarif.PhysicalLocation{ - ArtifactLocation: &sarif.ArtifactLocation{URI: &fileName}, - Region: &sarif.Region{ - StartLine: &startLine, - StartColumn: &startCol, - Snippet: &sarif.ArtifactContent{Text: &snippet}}}, +func TestGetRelativeLocationFileName(t *testing.T) { + tests := []struct { + location *sarif.Location + invocations []*sarif.Invocation + expectedOutput string + }{ + { + location: CreateLocation("file:///root/someDir/another/file", 1, 2, 3, 4, "snippet"), + invocations: []*sarif.Invocation{}, + expectedOutput: "root/someDir/another/file", + }, + { + location: CreateLocation("file:///root/someDir/another/file", 1, 2, 3, 4, "snippet"), + invocations: []*sarif.Invocation{{WorkingDirectory: sarif.NewSimpleArtifactLocation("/not/relevant")}}, + expectedOutput: "root/someDir/another/file", + }, + { + location: CreateLocation("file:///root/someDir/another/file", 1, 2, 3, 4, "snippet"), + invocations: []*sarif.Invocation{{WorkingDirectory: sarif.NewSimpleArtifactLocation("/root/someDir/")}}, + expectedOutput: "another/file", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetRelativeLocationFileName(test.location, test.invocations)) + } +} + +func TestGetFullLocationFileName(t *testing.T) { + tests := []struct { + file string + invocations []*sarif.Invocation + expectedOutput string + }{ + { + file: filepath.Join("root", "someDir", "another", "file"), + invocations: []*sarif.Invocation{}, + expectedOutput: filepath.Join("root", "someDir", "another", "file"), + }, + { + file: filepath.Join("another", "file"), + invocations: []*sarif.Invocation{ + {WorkingDirectory: sarif.NewSimpleArtifactLocation(filepath.Join("root", "someDir"))}, + {WorkingDirectory: sarif.NewSimpleArtifactLocation(filepath.Join("not", "relevant"))}, }, + expectedOutput: filepath.Join("root", "someDir", "another", "file"), + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetFullLocationFileName(test.file, test.invocations)) + } +} + +func TestSetLocationFileName(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput string + }{ + { + location: nil, + expectedOutput: "", + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: "changedFilename", + }, + } + + for _, test := range tests { + SetLocationFileName(test.location, test.expectedOutput) + assert.Equal(t, test.expectedOutput, GetLocationFileName(test.location)) + } +} + +func TestGetLocationRegion(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput *sarif.Region + }{ + { + location: nil, + expectedOutput: nil, + }, + { + location: &sarif.Location{PhysicalLocation: &sarif.PhysicalLocation{}}, + expectedOutput: nil, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: sarif.NewRegion().WithStartLine(1).WithStartColumn(2).WithEndLine(3).WithEndColumn(4). + WithSnippet(sarif.NewArtifactContent().WithText("snippet")), + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, getLocationRegion(test.location)) + } +} + +func TestGetLocationStartLine(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput int + }{ + { + location: nil, + expectedOutput: 0, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: 1, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationStartLine(test.location)) + } +} + +func TestGetLocationStartColumn(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput int + }{ + { + location: nil, + expectedOutput: 0, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: 2, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationStartColumn(test.location)) + } +} + +func TestGetLocationEndLine(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput int + }{ + { + location: nil, + expectedOutput: 0, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: 3, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationEndLine(test.location)) + } +} + +func TestGetLocationEndColumn(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput int + }{ + { + location: nil, + expectedOutput: 0, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: 4, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationEndColumn(test.location)) + } +} + +func TestExtractRelativePath(t *testing.T) { + tests := []struct { + fullPath string + projectPath string + expectedResult string + }{ + {fullPath: "file:///Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", + projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "tests/req.nodejs/file.js"}, + {fullPath: "file:///private/Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", + projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "tests/req.nodejs/file.js"}, + {fullPath: "invalidFullPath", + projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "invalidFullPath"}, + {fullPath: "", + projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: ""}, + {fullPath: "file:///Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", + projectPath: "invalidProjectPath", expectedResult: "Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js"}, + {fullPath: "file:///private/Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", + projectPath: "invalidProjectPath", expectedResult: "Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js"}, + } + + for _, test := range tests { + assert.Equal(t, test.expectedResult, ExtractRelativePath(test.fullPath, test.projectPath)) + } +} + +func TestGetResultSeverity(t *testing.T) { + levelValueHigh := string(errorLevel) + levelValueMedium := string(warningLevel) + levelValueMedium2 := string(infoLevel) + levelValueLow := string(noteLevel) + levelValueUnknown := string(noneLevel) + + tests := []struct { + result *sarif.Result + expectedSeverity string + }{ + {result: &sarif.Result{}, + expectedSeverity: "Medium"}, + {result: &sarif.Result{Level: &levelValueHigh}, + expectedSeverity: "High"}, + {result: &sarif.Result{Level: &levelValueMedium}, + expectedSeverity: "Medium"}, + {result: &sarif.Result{Level: &levelValueMedium2}, + expectedSeverity: "Medium"}, + {result: &sarif.Result{Level: &levelValueLow}, + expectedSeverity: "Low"}, + {result: &sarif.Result{Level: &levelValueUnknown}, + expectedSeverity: "Unknown"}, + } + + for _, test := range tests { + assert.Equal(t, test.expectedSeverity, GetResultSeverity(test.result)) + } +} + +func TestConvertToSarifLevel(t *testing.T) { + tests := []struct { + severity string + expectedOutput string + }{ + { + severity: "Unknown", + expectedOutput: "none", + }, + { + severity: "Low", + expectedOutput: "note", + }, + { + severity: "Medium", + expectedOutput: "warning", + }, + { + severity: "High", + expectedOutput: "error", + }, + { + severity: "Critical", + expectedOutput: "error", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, ConvertToSarifLevel(test.severity)) + } +} + +func TestIsApplicableResult(t *testing.T) { + tests := []struct { + name string + sarifResult *sarif.Result + expectedOutput bool + }{ + { + sarifResult: CreateDummyPassingResult("rule"), + expectedOutput: false, + }, + { + sarifResult: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet1", "ruleId1", "level1"), + expectedOutput: true, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, IsApplicableResult(test.sarifResult)) + } +} + +func TestGetRuleFullDescription(t *testing.T) { + tests := []struct { + rule *sarif.ReportingDescriptor + expectedOutput string + }{ + { + rule: sarif.NewRule("rule"), + expectedOutput: "", + }, + { + rule: sarif.NewRule("rule").WithFullDescription(nil), + expectedOutput: "", + }, + { + rule: sarif.NewRule("rule").WithFullDescription(sarif.NewMultiformatMessageString("description")), + expectedOutput: "description", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetRuleFullDescription(test.rule)) + } +} + +func TestCveToApplicabilityRuleId(t *testing.T) { + assert.Equal(t, "applic_cve", CveToApplicabilityRuleId("cve")) +} + +func TestApplicabilityRuleIdToCve(t *testing.T) { + tests := []struct { + ruleId string + expectedOutput string + }{ + { + ruleId: "rule", + expectedOutput: "rule", + }, + { + ruleId: "applic_cve", + expectedOutput: "cve", }, - Level: &level, - RuleID: &ruleId, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, ApplicabilityRuleIdToCve(test.ruleId)) + } +} + +func TestGetRunRules(t *testing.T) { + tests := []struct { + run *sarif.Run + expectedOutput []*sarif.ReportingDescriptor + }{ + { + run: &sarif.Run{}, + expectedOutput: []*sarif.ReportingDescriptor{}, + }, + { + run: CreateRunWithDummyResults(), + expectedOutput: []*sarif.ReportingDescriptor{}, + }, + { + run: CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + ), + expectedOutput: []*sarif.ReportingDescriptor{sarif.NewRule("rule1")}, + }, + { + run: CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + CreateDummyPassingResult("rule1"), + CreateDummyPassingResult("rule2"), + CreateDummyPassingResult("rule3"), + CreateDummyPassingResult("rule2"), + ), + expectedOutput: []*sarif.ReportingDescriptor{sarif.NewRule("rule1"), sarif.NewRule("rule2"), sarif.NewRule("rule3")}, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetRunRules(test.run)) + } +} + +func TestGetInvocationWorkingDirectory(t *testing.T) { + tests := []struct { + invocation *sarif.Invocation + expectedOutput string + }{ + { + invocation: nil, + expectedOutput: "", + }, + { + invocation: sarif.NewInvocation(), + expectedOutput: "", + }, + { + invocation: sarif.NewInvocation().WithWorkingDirectory(nil), + expectedOutput: "", + }, + { + invocation: sarif.NewInvocation().WithWorkingDirectory(sarif.NewArtifactLocation()), + expectedOutput: "", + }, + { + invocation: sarif.NewInvocation().WithWorkingDirectory(sarif.NewArtifactLocation().WithUri("file_to_wd")), + expectedOutput: "file_to_wd", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetInvocationWorkingDirectory(test.invocation)) } } diff --git a/xray/utils/test_sarifutils.go b/xray/utils/test_sarifutils.go new file mode 100644 index 000000000..5034b1eb6 --- /dev/null +++ b/xray/utils/test_sarifutils.go @@ -0,0 +1,64 @@ +package utils + +import "github.com/owenrumney/go-sarif/v2/sarif" + +func CreateRunWithDummyResults(results ...*sarif.Result) *sarif.Run { + run := sarif.NewRunWithInformationURI("", "") + for _, result := range results { + if result.RuleID != nil { + run.AddRule(*result.RuleID) + } + run.AddResult(result) + } + return run +} + +func CreateResultWithLocations(msg, ruleId, level string, locations ...*sarif.Location) *sarif.Result { + return &sarif.Result{ + Message: *sarif.NewTextMessage(msg), + Locations: locations, + Level: &level, + RuleID: &ruleId, + } +} + +func CreateLocation(fileName string, startLine, startCol, endLine, endCol int, snippet string) *sarif.Location { + return &sarif.Location{ + PhysicalLocation: &sarif.PhysicalLocation{ + ArtifactLocation: &sarif.ArtifactLocation{URI: &fileName}, + Region: &sarif.Region{ + StartLine: &startLine, + StartColumn: &startCol, + EndLine: &endLine, + EndColumn: &endCol, + Snippet: &sarif.ArtifactContent{Text: &snippet}}}, + } +} + +func CreateDummyPassingResult(ruleId string) *sarif.Result { + kind := "pass" + return &sarif.Result{ + Kind: &kind, + RuleID: &ruleId, + } +} + +func CreateResultWithOneLocation(fileName string, startLine, startCol, endLine, endCol int, snippet, ruleId, level string) *sarif.Result { + return CreateResultWithLocations("", ruleId, level, CreateLocation(fileName, startLine, startCol, endLine, endCol, snippet)) +} + +func CreateCodeFlow(threadFlows ...*sarif.ThreadFlow) *sarif.CodeFlow { + flow := sarif.NewCodeFlow() + for _, threadFlow := range threadFlows { + flow.AddThreadFlow(threadFlow) + } + return flow +} + +func CreateThreadFlow(locations ...*sarif.Location) *sarif.ThreadFlow { + stackStrace := sarif.NewThreadFlow() + for _, location := range locations { + stackStrace.AddLocation(sarif.NewThreadFlowLocation().WithLocation(location)) + } + return stackStrace +}