From e071ed5a828d52edb531db6b42dcaca93f185173 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Thu, 29 Feb 2024 07:43:12 +0200 Subject: [PATCH 1/5] Support release bundle creation by aql and artifacts --- artifactory/commands/generic/search.go | 26 +--- artifactory/utils/search.go | 28 ++++ go.mod | 2 +- go.sum | 4 +- lifecycle/createcommon.go | 169 ++++++++++++++++++++++++- lifecycle/createfromaql.go | 13 ++ lifecycle/createfromartifacts.go | 55 ++++++++ lifecycle/createfrombuilds.go | 71 +++++++++-- lifecycle/createfrombundles.go | 51 +++++++- 9 files changed, 378 insertions(+), 41 deletions(-) create mode 100644 lifecycle/createfromaql.go create mode 100644 lifecycle/createfromartifacts.go diff --git a/artifactory/commands/generic/search.go b/artifactory/commands/generic/search.go index cad7f1890..4c2aeecfb 100644 --- a/artifactory/commands/generic/search.go +++ b/artifactory/commands/generic/search.go @@ -1,6 +1,7 @@ package generic import ( + "errors" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" clientartutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" @@ -38,28 +39,15 @@ func (sc *SearchCommand) Search() (contentReader *content.ContentReader, err err } // Search Loop log.Info("Searching artifacts...") - var searchResults []*content.ContentReader + + searchResults, callbackFunc, err := utils.SearchFiles(servicesManager, sc.Spec()) defer func() { - for _, reader := range searchResults { - e := reader.Close() - if err == nil { - err = e - } - } + err = errors.Join(err, callbackFunc()) }() - for i := 0; i < len(sc.Spec().Files); i++ { - searchParams, err := utils.GetSearchParams(sc.Spec().Get(i)) - if err != nil { - log.Error(err) - return nil, err - } - reader, err := servicesManager.SearchFiles(searchParams) - if err != nil { - log.Error(err) - return nil, err - } - searchResults = append(searchResults, reader) + if err != nil { + return nil, err } + reader, err := utils.AqlResultToSearchResult(searchResults) if err != nil { return nil, err diff --git a/artifactory/utils/search.go b/artifactory/utils/search.go index a8888c82e..90866f2c1 100644 --- a/artifactory/utils/search.go +++ b/artifactory/utils/search.go @@ -2,6 +2,8 @@ package utils import ( "encoding/json" + "errors" + "github.com/jfrog/jfrog-client-go/artifactory" "github.com/jfrog/jfrog-cli-core/v2/common/spec" "github.com/jfrog/jfrog-client-go/artifactory/services" @@ -170,3 +172,29 @@ func SearchResultNoDate(reader *content.ContentReader) (contentReader *content.C contentReader = content.NewContentReader(writer.GetFilePath(), writer.GetArrayKey()) return } + +func SearchFiles(servicesManager artifactory.ArtifactoryServicesManager, spec *spec.SpecFiles) (searchResults []*content.ContentReader, callbackFunc func() error, err error) { + callbackFunc = func() error { + var errs error + for _, reader := range searchResults { + e := reader.Close() + errs = errors.Join(errs, e) + } + return err + } + + var curSearchParams services.SearchParams + var curReader *content.ContentReader + for i := 0; i < len(spec.Files); i++ { + curSearchParams, err = GetSearchParams(spec.Get(i)) + if err != nil { + return + } + curReader, err = servicesManager.SearchFiles(curSearchParams) + if err != nil { + return + } + searchResults = append(searchResults, curReader) + } + return +} diff --git a/go.mod b/go.mod index f80e75ea3..a9b177c88 100644 --- a/go.mod +++ b/go.mod @@ -96,7 +96,7 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240225150756-e5fed3788eca +replace github.com/jfrog/jfrog-client-go => github.com/RobiNino/jfrog-client-go v0.0.0-20240229054046-5812b410699d replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c diff --git a/go.sum b/go.sum index 7d2f662ec..62da2a712 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/RobiNino/jfrog-client-go v0.0.0-20240229054046-5812b410699d h1:dnmmSmJ6upXWhcujCN3SqE+pvTDD6NaRNjaXmKbgWy0= +github.com/RobiNino/jfrog-client-go v0.0.0-20240229054046-5812b410699d/go.mod h1:WhVrqiqhSNFwj58/RQIrJEd28PHH1LTD4eWE0vBXv1o= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -87,8 +89,6 @@ github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c h1:M1QiuCYGC github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c/go.mod h1:QHcKuesY4MrBVBuEwwBz4uIsX6mwYuMEDV09ng4AvAU= github.com/jfrog/gofrog v1.6.0 h1:jOwb37nHY2PnxePNFJ6e6279Pgkr3di05SbQQw47Mq8= github.com/jfrog/gofrog v1.6.0/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= -github.com/jfrog/jfrog-client-go v1.28.1-0.20240225150756-e5fed3788eca h1:wgiw3iokmQ5uK+6+M50fyMQBsMJPQEbRYnwbAMUmPlI= -github.com/jfrog/jfrog-client-go v1.28.1-0.20240225150756-e5fed3788eca/go.mod h1:WhVrqiqhSNFwj58/RQIrJEd28PHH1LTD4eWE0vBXv1o= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= diff --git a/lifecycle/createcommon.go b/lifecycle/createcommon.go index 4d5fba599..059fa73b3 100644 --- a/lifecycle/createcommon.go +++ b/lifecycle/createcommon.go @@ -1,11 +1,17 @@ package lifecycle import ( + "github.com/jfrog/jfrog-cli-core/v2/common/spec" "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-client-go/lifecycle/services" + "github.com/jfrog/jfrog-client-go/utils/errorutils" ) type ReleaseBundleCreateCommand struct { releaseBundleCmd + spec *spec.SpecFiles + // Backward compatibility: buildsSpecPath string releaseBundlesSpecPath string } @@ -44,11 +50,18 @@ func (rbc *ReleaseBundleCreateCommand) SetReleaseBundleProject(rbProjectKey stri return rbc } +func (rbc *ReleaseBundleCreateCommand) SetSpec(spec *spec.SpecFiles) *ReleaseBundleCreateCommand { + rbc.spec = spec + return rbc +} + +// Deprecated func (rbc *ReleaseBundleCreateCommand) SetBuildsSpecPath(buildsSpecPath string) *ReleaseBundleCreateCommand { rbc.buildsSpecPath = buildsSpecPath return rbc } +// Deprecated func (rbc *ReleaseBundleCreateCommand) SetReleaseBundlesSpecPath(releaseBundlesSpecPath string) *ReleaseBundleCreateCommand { rbc.releaseBundlesSpecPath = releaseBundlesSpecPath return rbc @@ -72,8 +85,160 @@ func (rbc *ReleaseBundleCreateCommand) Run() error { return err } - if rbc.buildsSpecPath != "" { + sourceType, err := rbc.identifySourceType() + if err != nil { + return err + } + + switch sourceType { + case services.Aql: + return rbc.createFromAql(servicesManager, rbDetails, queryParams) + case services.Artifacts: + return rbc.createFromArtifacts(servicesManager, rbDetails, queryParams) + case services.Builds: return rbc.createFromBuilds(servicesManager, rbDetails, queryParams) + case services.ReleaseBundles: + return rbc.createFromReleaseBundles(servicesManager, rbDetails, queryParams) + default: + return errorutils.CheckErrorf("unknown source for release bundle creation was provided") + } +} + +func (rbc *ReleaseBundleCreateCommand) identifySourceType() (services.SourceType, error) { + switch { + case rbc.buildsSpecPath != "": + return services.Builds, nil + case rbc.releaseBundlesSpecPath != "": + return services.ReleaseBundles, nil + case rbc.spec != nil: + return validateAndIdentifyRbCreationSpec(rbc.spec.Files) + default: + return "", errorutils.CheckErrorf("a spec file input is mandatory") + } +} + +func validateAndIdentifyRbCreationSpec(files []spec.File) (services.SourceType, error) { + if len(files) == 0 { + return "", errorutils.CheckErrorf("spec must include at least one file group") + } + + var detectedCreationSources []services.SourceType + for _, file := range files { + sourceType, err := validateFile(file) + if err != nil { + return "", err + } + detectedCreationSources = append(detectedCreationSources, sourceType) + } + + if err := validateCreationSources(detectedCreationSources); err != nil { + return "", err + } + return detectedCreationSources[0], nil +} + +func validateCreationSources(detectedCreationSources []services.SourceType) error { + if len(detectedCreationSources) == 0 { + return errorutils.CheckErrorf("unexpected err while validating spec - could not detect any creation sources") + } + + // Assert single creation source. + for i := 1; i < len(detectedCreationSources); i++ { + if detectedCreationSources[i] != detectedCreationSources[0] { + return generateSingleCreationSourceErr(detectedCreationSources) + } + } + + // If aql, assert single file. + if detectedCreationSources[0] == services.Aql && len(detectedCreationSources) > 1 { + return errorutils.CheckErrorf("only a single aql query can be provided") + } + return nil +} + +func generateSingleCreationSourceErr(detectedCreationSources []services.SourceType) error { + var detectedStr []string + for _, source := range detectedCreationSources { + detectedStr = append(detectedStr, string(source)) + } + return errorutils.CheckErrorf( + "multiple creation sources were detected in separate spec files. Only a single creation source should be provided. Detected: '%s'", + coreutils.ListToText(detectedStr)) +} + +func validateFile(file spec.File) (services.SourceType, error) { + // Aql creation source: + isAql := len(file.Aql.ItemsFind) > 0 + + // Build creation source: + isBuild := len(file.Build) > 0 + isIncludeDeps, _ := file.IsIncludeDeps(false) + + // Bundle creation source: + isBundle := len(file.Bundle) > 0 + + // Build & bundle: + isProject := len(file.Project) > 0 + + // Artifacts creation source: + isPattern := len(file.Pattern) > 0 + isExclusions := len(file.Exclusions) > 0 && len(file.Exclusions[0]) > 0 + isProps := len(file.Props) > 0 + isExcludeProps := len(file.ExcludeProps) > 0 + isRecursive, _ := file.IsRecursive(true) + + // Unsupported: + isPathMapping := len(file.PathMapping.Input) > 0 && len(file.PathMapping.Output) > 0 + isTarget := len(file.Target) > 0 + isSortOrder := len(file.SortOrder) > 0 + isSortBy := len(file.SortBy) > 0 + isExcludeArtifacts, _ := file.IsExcludeArtifacts(false) + isGPGKey := len(file.PublicGpgKey) > 0 + isOffset := file.Offset > 0 + isLimit := file.Limit > 0 + isArchive := len(file.Archive) > 0 + isSymlinks, _ := file.IsSymlinks(false) + isRegexp := file.Regexp == "true" + isAnt := file.Ant == "true" + isExplode, _ := file.IsExplode(false) + isBypassArchiveInspection, _ := file.IsBypassArchiveInspection(false) + isTransitive, _ := file.IsTransitive(false) + + if isPathMapping || isTarget || isSortOrder || isSortBy || isExcludeArtifacts || isGPGKey || isOffset || isLimit || + isSymlinks || isArchive || isAnt || isRegexp || isExplode || isBypassArchiveInspection || isTransitive { + return "", errorutils.CheckErrorf("unsupported fields were provided in file spec. " + + "release bundle creation file spec only supports the following fields: " + + "'aql', 'build', 'includeDeps', 'bundle', 'project', 'pattern', 'exclusions', 'props', 'excludeProps' and 'recursive'") + } + if coreutils.SumTrueValues([]bool{isAql, isBuild, isBundle, isPattern}) != 1 { + return "", errorutils.CheckErrorf("exactly one creation source is supported (aql, builds, release bundles or pattern (artifacts))") + } + + switch { + case isAql: + return services.Aql, + validateCreationSource([]bool{isIncludeDeps, isProject, isExclusions, isProps, isExcludeProps, !isRecursive}, + "aql creation source supports no other fields") + case isBuild: + return services.Builds, + validateCreationSource([]bool{isExclusions, isProps, isExcludeProps, !isRecursive}, + "builds creation source only supports the 'includeDeps' and 'project' fields") + case isBundle: + return services.ReleaseBundles, + validateCreationSource([]bool{isIncludeDeps, isExclusions, isProps, isExcludeProps, !isRecursive}, + "release bundles creation source only supports the 'project' field") + case isPattern: + return services.Artifacts, + validateCreationSource([]bool{isIncludeDeps, isProject}, + "release bundles creation source only supports the 'exclusions', 'props', 'excludeProps' and 'recursive' fields") + default: + return "", errorutils.CheckErrorf("unexpected err in spec validation") + } +} + +func validateCreationSource(unsupportedFields []bool, errMsg string) error { + if coreutils.SumTrueValues(unsupportedFields) > 0 { + return errorutils.CheckErrorf(errMsg) } - return rbc.createFromReleaseBundles(servicesManager, rbDetails, queryParams) + return nil } diff --git a/lifecycle/createfromaql.go b/lifecycle/createfromaql.go new file mode 100644 index 000000000..12918bd75 --- /dev/null +++ b/lifecycle/createfromaql.go @@ -0,0 +1,13 @@ +package lifecycle + +import ( + "fmt" + "github.com/jfrog/jfrog-client-go/lifecycle" + "github.com/jfrog/jfrog-client-go/lifecycle/services" +) + +func (rbc *ReleaseBundleCreateCommand) createFromAql(servicesManager *lifecycle.LifecycleServicesManager, + rbDetails services.ReleaseBundleDetails, queryParams services.CommonOptionalQueryParams) error { + aqlQuery := fmt.Sprintf(`items.find(%s)`, rbc.spec.Get(0).Aql.ItemsFind) + return servicesManager.CreateReleaseBundleFromAql(rbDetails, queryParams, rbc.signingKeyName, aqlQuery) +} diff --git a/lifecycle/createfromartifacts.go b/lifecycle/createfromartifacts.go new file mode 100644 index 000000000..8a2633451 --- /dev/null +++ b/lifecycle/createfromartifacts.go @@ -0,0 +1,55 @@ +package lifecycle + +import ( + "errors" + "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" + rtServicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/lifecycle" + "github.com/jfrog/jfrog-client-go/lifecycle/services" + "github.com/jfrog/jfrog-client-go/utils/io/content" + "github.com/jfrog/jfrog-client-go/utils/log" + "path" +) + +func (rbc *ReleaseBundleCreateCommand) createFromArtifacts(lcServicesManager *lifecycle.LifecycleServicesManager, + rbDetails services.ReleaseBundleDetails, queryParams services.CommonOptionalQueryParams) (err error) { + + rtServicesManager, err := utils.CreateServiceManager(rbc.serverDetails, 3, 0, false) + if err != nil { + return err + } + + log.Info("Searching artifacts...") + searchResults, callbackFunc, err := utils.SearchFiles(rtServicesManager, rbc.spec) + defer func() { + err = errors.Join(err, callbackFunc()) + }() + if err != nil { + return err + } + artifactsSource, err := aqlResultToArtifactsSource(searchResults) + if err != nil { + return err + } + + return lcServicesManager.CreateReleaseBundleFromArtifacts(rbDetails, queryParams, rbc.signingKeyName, artifactsSource) +} + +func aqlResultToArtifactsSource(readers []*content.ContentReader) (artifactsSource services.CreateFromArtifacts, err error) { + for _, reader := range readers { + for searchResult := new(rtServicesUtils.ResultItem); reader.NextRecord(searchResult) == nil; searchResult = new(rtServicesUtils.ResultItem) { + if err != nil { + return + } + artifactsSource.Artifacts = append(artifactsSource.Artifacts, services.ArtifactSource{ + Path: path.Join(searchResult.Repo, searchResult.Path, searchResult.Name), + Sha256: searchResult.Sha256, + }) + } + if err = reader.GetError(); err != nil { + return + } + reader.Reset() + } + return +} diff --git a/lifecycle/createfrombuilds.go b/lifecycle/createfrombuilds.go index a955b997a..02e54e644 100644 --- a/lifecycle/createfrombuilds.go +++ b/lifecycle/createfrombuilds.go @@ -3,6 +3,7 @@ package lifecycle import ( "encoding/json" rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" + "github.com/jfrog/jfrog-cli-core/v2/common/spec" rtServices "github.com/jfrog/jfrog-client-go/artifactory/services" "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/lifecycle" @@ -14,27 +15,38 @@ import ( func (rbc *ReleaseBundleCreateCommand) createFromBuilds(servicesManager *lifecycle.LifecycleServicesManager, rbDetails services.ReleaseBundleDetails, queryParams services.CommonOptionalQueryParams) error { - builds := CreateFromBuildsSpec{} - content, err := fileutils.ReadFile(rbc.buildsSpecPath) + var buildsSource services.CreateFromBuildsSource + var err error + if rbc.buildsSpecPath != "" { + buildsSource, err = rbc.getBuildSourceFromBuildsSpec() + } else { + buildsSource, err = rbc.convertSpecToBuildsSource(rbc.spec.Files) + } if err != nil { return err } - if err = json.Unmarshal(content, &builds); err != nil { - return errorutils.CheckError(err) - } - if len(builds.Builds) == 0 { + if len(buildsSource.Builds) == 0 { return errorutils.CheckErrorf("at least one build is expected in order to create a release bundle from builds") } - buildsSource, err := rbc.convertToBuildsSource(builds) + return servicesManager.CreateReleaseBundleFromBuilds(rbDetails, queryParams, rbc.signingKeyName, buildsSource) +} + +func (rbc *ReleaseBundleCreateCommand) getBuildSourceFromBuildsSpec() (buildsSource services.CreateFromBuildsSource, err error) { + builds := CreateFromBuildsSpec{} + content, err := fileutils.ReadFile(rbc.buildsSpecPath) if err != nil { - return err + return } - return servicesManager.CreateReleaseBundleFromBuilds(rbDetails, queryParams, rbc.signingKeyName, buildsSource) + if err = json.Unmarshal(content, &builds); errorutils.CheckError(err) != nil { + return + } + + return rbc.convertBuildsSpecToBuildsSource(builds) } -func (rbc *ReleaseBundleCreateCommand) convertToBuildsSource(builds CreateFromBuildsSpec) (services.CreateFromBuildsSource, error) { +func (rbc *ReleaseBundleCreateCommand) convertBuildsSpecToBuildsSource(builds CreateFromBuildsSpec) (services.CreateFromBuildsSource, error) { buildsSource := services.CreateFromBuildsSource{} for _, build := range builds.Builds { buildSource := services.BuildSource{BuildName: build.Name, IncludeDependencies: build.IncludeDependencies} @@ -49,6 +61,29 @@ func (rbc *ReleaseBundleCreateCommand) convertToBuildsSource(builds CreateFromBu return buildsSource, nil } +func (rbc *ReleaseBundleCreateCommand) convertSpecToBuildsSource(files []spec.File) (services.CreateFromBuildsSource, error) { + buildsSource := services.CreateFromBuildsSource{} + for _, file := range files { + buildName, buildNumber, err := rbc.getBuildDetailsFromIdentifier(file.Build, file.Project) + if err != nil { + return services.CreateFromBuildsSource{}, err + } + isIncludeDeps, err := file.IsIncludeDeps(false) + if err != nil { + return services.CreateFromBuildsSource{}, err + } + + buildSource := services.BuildSource{ + BuildName: buildName, + BuildNumber: buildNumber, + BuildRepository: utils.GetBuildInfoRepositoryByProject(file.Project), + IncludeDependencies: isIncludeDeps, + } + buildsSource.Builds = append(buildsSource.Builds, buildSource) + } + return buildsSource, nil +} + func (rbc *ReleaseBundleCreateCommand) getLatestBuildNumberIfEmpty(buildName, buildNumber, project string) (string, error) { if buildNumber != "" { return buildNumber, nil @@ -69,6 +104,22 @@ func (rbc *ReleaseBundleCreateCommand) getLatestBuildNumberIfEmpty(buildName, bu return buildNumber, nil } +func (rbc *ReleaseBundleCreateCommand) getBuildDetailsFromIdentifier(buildIdentifier, project string) (string, string, error) { + aqlService, err := rbc.getAqlService() + if err != nil { + return "", "", err + } + + buildName, buildNumber, err := utils.GetBuildNameAndNumberFromBuildIdentifier(buildIdentifier, project, aqlService) + if err != nil { + return "", "", err + } + if buildName == "" || buildNumber == "" { + return "", "", errorutils.CheckErrorf("could not identify a build info by the '%s' identifier in artifactory", buildIdentifier) + } + return buildName, buildNumber, nil +} + func (rbc *ReleaseBundleCreateCommand) getAqlService() (*rtServices.AqlService, error) { rtServiceManager, err := rtUtils.CreateServiceManager(rbc.serverDetails, 3, 0, false) if err != nil { diff --git a/lifecycle/createfrombundles.go b/lifecycle/createfrombundles.go index 1e9253351..7c18141de 100644 --- a/lifecycle/createfrombundles.go +++ b/lifecycle/createfrombundles.go @@ -2,6 +2,8 @@ package lifecycle import ( "encoding/json" + "github.com/jfrog/jfrog-cli-core/v2/common/spec" + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/lifecycle" "github.com/jfrog/jfrog-client-go/lifecycle/services" "github.com/jfrog/jfrog-client-go/utils/errorutils" @@ -11,20 +13,21 @@ import ( func (rbc *ReleaseBundleCreateCommand) createFromReleaseBundles(servicesManager *lifecycle.LifecycleServicesManager, rbDetails services.ReleaseBundleDetails, queryParams services.CommonOptionalQueryParams) error { - bundles := CreateFromReleaseBundlesSpec{} - content, err := fileutils.ReadFile(rbc.releaseBundlesSpecPath) + var releaseBundlesSource services.CreateFromReleaseBundlesSource + var err error + if rbc.releaseBundlesSpecPath != "" { + releaseBundlesSource, err = rbc.getReleaseBundlesSourceFromBundlesSpec() + } else { + releaseBundlesSource, err = rbc.convertSpecToReleaseBundlesSource(rbc.spec.Files) + } if err != nil { return err } - if err = json.Unmarshal(content, &bundles); err != nil { - return errorutils.CheckError(err) - } - if len(bundles.ReleaseBundles) == 0 { + if len(releaseBundlesSource.ReleaseBundles) == 0 { return errorutils.CheckErrorf("at least one release bundle is expected in order to create a release bundle from release bundles") } - releaseBundlesSource := rbc.convertToReleaseBundlesSource(bundles) return servicesManager.CreateReleaseBundleFromBundles(rbDetails, queryParams, rbc.signingKeyName, releaseBundlesSource) } @@ -41,6 +44,40 @@ func (rbc *ReleaseBundleCreateCommand) convertToReleaseBundlesSource(bundles Cre return releaseBundlesSource } +func (rbc *ReleaseBundleCreateCommand) convertSpecToReleaseBundlesSource(files []spec.File) (services.CreateFromReleaseBundlesSource, error) { + releaseBundlesSource := services.CreateFromReleaseBundlesSource{} + for _, file := range files { + name, version, err := utils.ParseNameAndVersion(file.Bundle, false) + if err != nil { + return releaseBundlesSource, err + } + if name == "" || version == "" { + return releaseBundlesSource, errorutils.CheckErrorf( + "invalid release bundle source was provided. Both name and version are mandatory. Provided name: '%s', version: '%s'", name, version) + } + rbSource := services.ReleaseBundleSource{ + ReleaseBundleName: name, + ReleaseBundleVersion: version, + ProjectKey: file.Project, + } + releaseBundlesSource.ReleaseBundles = append(releaseBundlesSource.ReleaseBundles, rbSource) + } + return releaseBundlesSource, nil +} + +func (rbc *ReleaseBundleCreateCommand) getReleaseBundlesSourceFromBundlesSpec() (releaseBundlesSource services.CreateFromReleaseBundlesSource, err error) { + releaseBundles := CreateFromReleaseBundlesSpec{} + content, err := fileutils.ReadFile(rbc.releaseBundlesSpecPath) + if err != nil { + return + } + if err = json.Unmarshal(content, &releaseBundles); errorutils.CheckError(err) != nil { + return + } + + return rbc.convertToReleaseBundlesSource(releaseBundles), nil +} + type CreateFromReleaseBundlesSpec struct { ReleaseBundles []SourceReleaseBundleSpec `json:"releaseBundles,omitempty"` } From cf0dd84cc9bea3399d0f56541ac476f1e52ecf6a Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Mon, 18 Mar 2024 11:00:18 +0200 Subject: [PATCH 2/5] cr comments --- artifactory/utils/search.go | 5 +-- go.mod | 2 +- go.sum | 4 +- lifecycle/createcommon.go | 20 ++++++--- lifecycle/createcommon_test.go | 76 ++++++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 12 deletions(-) create mode 100644 lifecycle/createcommon_test.go diff --git a/artifactory/utils/search.go b/artifactory/utils/search.go index 90866f2c1..1ed5823a5 100644 --- a/artifactory/utils/search.go +++ b/artifactory/utils/search.go @@ -177,10 +177,9 @@ func SearchFiles(servicesManager artifactory.ArtifactoryServicesManager, spec *s callbackFunc = func() error { var errs error for _, reader := range searchResults { - e := reader.Close() - errs = errors.Join(errs, e) + errs = errors.Join(errs, reader.Close()) } - return err + return errs } var curSearchParams services.SearchParams diff --git a/go.mod b/go.mod index eaefca790..bfebd6856 100644 --- a/go.mod +++ b/go.mod @@ -96,7 +96,7 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/RobiNino/jfrog-client-go v0.0.0-20240229054046-5812b410699d +replace github.com/jfrog/jfrog-client-go => github.com/RobiNino/jfrog-client-go v0.0.0-20240318071709-a3702cde23d6 replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c diff --git a/go.sum b/go.sum index 09a845537..71ca50eed 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/RobiNino/jfrog-client-go v0.0.0-20240229054046-5812b410699d h1:dnmmSmJ6upXWhcujCN3SqE+pvTDD6NaRNjaXmKbgWy0= -github.com/RobiNino/jfrog-client-go v0.0.0-20240229054046-5812b410699d/go.mod h1:WhVrqiqhSNFwj58/RQIrJEd28PHH1LTD4eWE0vBXv1o= +github.com/RobiNino/jfrog-client-go v0.0.0-20240318071709-a3702cde23d6 h1:h3+sk0tylIVX1p1VsHAjRX8hfkg/p9xE/+MMGOHBYCE= +github.com/RobiNino/jfrog-client-go v0.0.0-20240318071709-a3702cde23d6/go.mod h1:NB8tYFgkWtn+wHsKC+aYC75aLnS6yW81d8JAFTBxsi0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= diff --git a/lifecycle/createcommon.go b/lifecycle/createcommon.go index fc1e5cba6..6b0e7f35c 100644 --- a/lifecycle/createcommon.go +++ b/lifecycle/createcommon.go @@ -8,6 +8,12 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" ) +const ( + missingCreationSourcesErrMsg = "unexpected err while validating spec - could not detect any creation sources" + multipleCreationSourcesErrMsg = "multiple creation sources were detected in separate spec files. Only a single creation source should be provided. Detected:" + singleAqlErrMsg = "only a single aql query can be provided" +) + type ReleaseBundleCreateCommand struct { releaseBundleCmd signingKeyName string @@ -140,7 +146,7 @@ func validateAndIdentifyRbCreationSpec(files []spec.File) (services.SourceType, func validateCreationSources(detectedCreationSources []services.SourceType) error { if len(detectedCreationSources) == 0 { - return errorutils.CheckErrorf("unexpected err while validating spec - could not detect any creation sources") + return errorutils.CheckErrorf(missingCreationSourcesErrMsg) } // Assert single creation source. @@ -152,7 +158,7 @@ func validateCreationSources(detectedCreationSources []services.SourceType) erro // If aql, assert single file. if detectedCreationSources[0] == services.Aql && len(detectedCreationSources) > 1 { - return errorutils.CheckErrorf("only a single aql query can be provided") + return errorutils.CheckErrorf(singleAqlErrMsg) } return nil } @@ -163,8 +169,7 @@ func generateSingleCreationSourceErr(detectedCreationSources []services.SourceTy detectedStr = append(detectedStr, string(source)) } return errorutils.CheckErrorf( - "multiple creation sources were detected in separate spec files. Only a single creation source should be provided. Detected: '%s'", - coreutils.ListToText(detectedStr)) + "%s '%s'", multipleCreationSourcesErrMsg, coreutils.ListToText(detectedStr)) } func validateFile(file spec.File) (services.SourceType, error) { @@ -186,10 +191,13 @@ func validateFile(file spec.File) (services.SourceType, error) { isExclusions := len(file.Exclusions) > 0 && len(file.Exclusions[0]) > 0 isProps := len(file.Props) > 0 isExcludeProps := len(file.ExcludeProps) > 0 - isRecursive, _ := file.IsRecursive(true) + isRecursive, err := file.IsRecursive(true) + if err != nil { + return "", errorutils.CheckErrorf("invalid value provided to the 'recursive' field. error: %s", err.Error()) + } // Unsupported: - isPathMapping := len(file.PathMapping.Input) > 0 && len(file.PathMapping.Output) > 0 + isPathMapping := len(file.PathMapping.Input) > 0 || len(file.PathMapping.Output) > 0 isTarget := len(file.Target) > 0 isSortOrder := len(file.SortOrder) > 0 isSortBy := len(file.SortBy) > 0 diff --git a/lifecycle/createcommon_test.go b/lifecycle/createcommon_test.go new file mode 100644 index 000000000..a0c3b598f --- /dev/null +++ b/lifecycle/createcommon_test.go @@ -0,0 +1,76 @@ +package lifecycle + +import ( + "github.com/jfrog/jfrog-cli-core/v2/common/spec" + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/lifecycle/services" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestValidateCreationSources(t *testing.T) { + testCases := []struct { + testName string + detectedCreationSources []services.SourceType + errExpected bool + errMsg string + }{ + {"missing creation sources", []services.SourceType{}, true, missingCreationSourcesErrMsg}, + {"single creation source", []services.SourceType{services.Aql, services.Artifacts, services.Builds}, + true, multipleCreationSourcesErrMsg + " 'aql, artifacts and builds'"}, + {"single aql err", []services.SourceType{services.Aql, services.Aql}, true, singleAqlErrMsg}, + {"valid aql", []services.SourceType{services.Aql}, false, ""}, + {"valid artifacts", []services.SourceType{services.Artifacts, services.Artifacts}, false, ""}, + {"valid builds", []services.SourceType{services.Builds, services.Builds}, false, ""}, + {"valid release bundles", []services.SourceType{services.ReleaseBundles, services.ReleaseBundles}, false, ""}, + } + for _, testCase := range testCases { + t.Run(testCase.testName, func(t *testing.T) { + err := validateCreationSources(testCase.detectedCreationSources) + if testCase.errExpected { + assert.EqualError(t, err, testCase.errMsg) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateFile(t *testing.T) { + testCases := []struct { + testName string + file spec.File + errExpected bool + expectedSourceType services.SourceType + }{ + {"valid aql", spec.File{Aql: utils.Aql{ItemsFind: "abc"}}, false, services.Aql}, + {"valid build", spec.File{Build: "name/number", IncludeDeps: "true", Project: "project"}, false, services.Builds}, + {"valid bundle", spec.File{Bundle: "name/number", Project: "project"}, false, services.ReleaseBundles}, + {"valid artifacts", + spec.File{ + Pattern: "repo/path/file", + Exclusions: []string{"exclude"}, + Props: "prop", + ExcludeProps: "exclude prop", + Recursive: "false"}, false, services.Artifacts}, + {"invalid fields", spec.File{PathMapping: utils.PathMapping{Input: "input"}, Target: "target"}, true, ""}, + {"multiple creation sources", + spec.File{Aql: utils.Aql{ItemsFind: "abc"}, Build: "name/number", Bundle: "name/number", Pattern: "repo/path/file"}, + true, ""}, + {"invalid aql", spec.File{Aql: utils.Aql{ItemsFind: "abc"}, Props: "prop"}, true, ""}, + {"invalid builds", spec.File{Build: "name/number", Recursive: "false"}, true, ""}, + {"invalid bundles", spec.File{Bundle: "name/number", IncludeDeps: "true"}, true, ""}, + {"invalid artifacts", spec.File{Pattern: "repo/path/file", Project: "proj"}, true, ""}, + } + for _, testCase := range testCases { + t.Run(testCase.testName, func(t *testing.T) { + sourceType, err := validateFile(testCase.file) + if testCase.errExpected { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, testCase.expectedSourceType, sourceType) + } + }) + } +} From 05027ac856490a96f9e35822ebf7442825bd9a65 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Mon, 18 Mar 2024 12:43:23 +0200 Subject: [PATCH 3/5] styling --- artifactory/commands/generic/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifactory/commands/generic/search.go b/artifactory/commands/generic/search.go index 4c2aeecfb..b5cf9ff40 100644 --- a/artifactory/commands/generic/search.go +++ b/artifactory/commands/generic/search.go @@ -27,7 +27,7 @@ func (sc *SearchCommand) Run() error { return err } -func (sc *SearchCommand) Search() (contentReader *content.ContentReader, err error) { +func (sc *SearchCommand) Search() (*content.ContentReader, error) { // Service Manager serverDetails, err := sc.ServerDetails() if errorutils.CheckError(err) != nil { From 992f3cf7d63b3aebd6e61d943e431a55e6d5b860 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Mon, 18 Mar 2024 13:18:56 +0200 Subject: [PATCH 4/5] styling --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bfebd6856..5d072cb22 100644 --- a/go.mod +++ b/go.mod @@ -96,7 +96,7 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/RobiNino/jfrog-client-go v0.0.0-20240318071709-a3702cde23d6 +replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240318111807-75c3311549ab replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c From 30c59deaaeaa76f9b04b43e380fec932c1ea8bc7 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Mon, 18 Mar 2024 13:28:09 +0200 Subject: [PATCH 5/5] go.sum --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 71ca50eed..6bdaf3eda 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,6 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/RobiNino/jfrog-client-go v0.0.0-20240318071709-a3702cde23d6 h1:h3+sk0tylIVX1p1VsHAjRX8hfkg/p9xE/+MMGOHBYCE= -github.com/RobiNino/jfrog-client-go v0.0.0-20240318071709-a3702cde23d6/go.mod h1:NB8tYFgkWtn+wHsKC+aYC75aLnS6yW81d8JAFTBxsi0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -89,6 +87,8 @@ github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c h1:M1QiuCYGC github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c/go.mod h1:QHcKuesY4MrBVBuEwwBz4uIsX6mwYuMEDV09ng4AvAU= github.com/jfrog/gofrog v1.6.3 h1:F7He0+75HcgCe6SGTSHLFCBDxiE2Ja0tekvvcktW6wc= github.com/jfrog/gofrog v1.6.3/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= +github.com/jfrog/jfrog-client-go v1.28.1-0.20240318111807-75c3311549ab h1:A5VTYC5x9swhypyJt2j47WBRPiJjn+0mc7DMgcmdiSE= +github.com/jfrog/jfrog-client-go v1.28.1-0.20240318111807-75c3311549ab/go.mod h1:NB8tYFgkWtn+wHsKC+aYC75aLnS6yW81d8JAFTBxsi0= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=