Support release bundle creation by aql and artifacts

artifactory_test.go

@@ -4521,7 +4521,7 @@ func TestArtifactoryDeleteExcludeProps(t *testing.T) {
 	initArtifactoryTest(t, "")
 
 	// Upload files
-	specFile, err := tests.CreateSpec(tests.UploadWithPropsSpecdeleteExcludeProps)
+	specFile, err := tests.CreateSpec(tests.UploadWithPropsSpecDeleteExcludeProps)
 	assert.NoError(t, err)
 	runRt(t, "upload", "--spec="+specFile, "--recursive")
 

go.mod

@@ -130,9 +130,9 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240225151519-de88b95a3824
+replace github.com/jfrog/jfrog-cli-core/v2 => github.com/RobiNino/jfrog-cli-core/v2 v2.0.0-20240229054312-e071ed5a828d
 
-replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240225150756-e5fed3788eca
+replace github.com/jfrog/jfrog-client-go => github.com/RobiNino/jfrog-client-go v0.0.0-20240229054046-5812b410699d
 
 replace github.com/jfrog/jfrog-cli-security => github.com/jfrog/jfrog-cli-security v1.0.4-0.20240225141439-cc8b9f3d1962
 

go.sum

@@ -14,6 +14,10 @@ github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7
 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
 github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
 github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
+github.com/RobiNino/jfrog-cli-core/v2 v2.0.0-20240229054312-e071ed5a828d h1:GfDrWWRsBbq/zXZtgwtnJPuI4+YiZ8TeTXguuA+28ps=
+github.com/RobiNino/jfrog-cli-core/v2 v2.0.0-20240229054312-e071ed5a828d/go.mod h1:PVLAo8E9/FqTL9OULieObfrvI/2sF7d+TekAgnHFr9Y=
+github.com/RobiNino/jfrog-client-go v0.0.0-20240229054046-5812b410699d h1:dnmmSmJ6upXWhcujCN3SqE+pvTDD6NaRNjaXmKbgWy0=
+github.com/RobiNino/jfrog-client-go v0.0.0-20240229054046-5812b410699d/go.mod h1:WhVrqiqhSNFwj58/RQIrJEd28PHH1LTD4eWE0vBXv1o=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
@@ -127,12 +131,8 @@ github.com/jfrog/gofrog v1.6.0 h1:jOwb37nHY2PnxePNFJ6e6279Pgkr3di05SbQQw47Mq8=
 github.com/jfrog/gofrog v1.6.0/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg=
 github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=
 github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w=
-github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240225151519-de88b95a3824 h1:nSqx+EdsuIzKsjS/5dkRaD9ak4db9IYMf/ggfCoahAE=
-github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240225151519-de88b95a3824/go.mod h1:aE5kYuqiZxu6hHkAQm34BvtGjLR8rk0/PUWpl4u5g0Q=
 github.com/jfrog/jfrog-cli-security v1.0.4-0.20240225141439-cc8b9f3d1962 h1:dlVDm3q/gR/yHrTcABHjMucs5vT6JesEw6GwBQe4WK4=
 github.com/jfrog/jfrog-cli-security v1.0.4-0.20240225141439-cc8b9f3d1962/go.mod h1:u2j9vEorky4zk7EWk+Bqw08o66p9tuW7LxVxW0boAMA=
-github.com/jfrog/jfrog-client-go v1.28.1-0.20240225150756-e5fed3788eca h1:wgiw3iokmQ5uK+6+M50fyMQBsMJPQEbRYnwbAMUmPlI=
-github.com/jfrog/jfrog-client-go v1.28.1-0.20240225150756-e5fed3788eca/go.mod h1:WhVrqiqhSNFwj58/RQIrJEd28PHH1LTD4eWE0vBXv1o=
 github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI=
 github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=

lifecycle/cli.go

@@ -4,9 +4,11 @@ import (
 	"errors"
 	commonCliUtils "github.com/jfrog/jfrog-cli-core/v2/common/cliutils"
 	"github.com/jfrog/jfrog-cli-core/v2/common/commands"
+	"github.com/jfrog/jfrog-cli-core/v2/common/spec"
 	coreCommon "github.com/jfrog/jfrog-cli-core/v2/docs/common"
 	"github.com/jfrog/jfrog-cli-core/v2/lifecycle"
 	coreConfig "github.com/jfrog/jfrog-cli-core/v2/utils/config"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"github.com/jfrog/jfrog-cli/docs/common"
 	rbCreate "github.com/jfrog/jfrog-cli/docs/lifecycle/create"
 	rbDistribute "github.com/jfrog/jfrog-cli/docs/lifecycle/distribute"
@@ -75,10 +77,19 @@ func validateCreateReleaseBundleContext(c *cli.Context) error {
 		return err
 	}
 
-	bothProvided := c.IsSet(cliutils.Builds) && c.IsSet(cliutils.ReleaseBundles)
-	noneProvided := !c.IsSet(cliutils.Builds) && !c.IsSet(cliutils.ReleaseBundles)
-	if bothProvided || noneProvided {
-		return errorutils.CheckErrorf("exactly one of the following options must be supplied: --%s or --%s", cliutils.Builds, cliutils.ReleaseBundles)
+	return assertValidCreationMethod(c)
+}
+
+func assertValidCreationMethod(c *cli.Context) error {
+	methods := []bool{
+		c.IsSet("spec"), c.IsSet(cliutils.Builds), c.IsSet(cliutils.ReleaseBundles)}
+	if coreutils.SumTrueValues(methods) > 1 {
+		return errorutils.CheckErrorf("exactly one creation source must be supplied: --%s, --%s or --%s.\n"+
+			"The spec option is the recommended approach.", "spec", cliutils.Builds, cliutils.ReleaseBundles)
+	}
+	// If the user did not provide a source, we suggest only the recommended spec approach.
+	if coreutils.SumTrueValues(methods) == 0 {
+		return errorutils.CheckErrorf("the --spec option is mandatory")
 	}
 	return nil
 }
@@ -88,15 +99,23 @@ func create(c *cli.Context) (err error) {
 		return err
 	}
 
+	var creationSpec *spec.SpecFiles
+	if c.IsSet("spec") {
+		creationSpec, err = cliutils.GetSpec(c, true)
+		if err != nil {
+			return
+		}
+	}
+
 	lcDetails, err := createLifecycleDetailsByFlags(c)
 	if err != nil {
 		return
 	}
 
 	createCmd := lifecycle.NewReleaseBundleCreateCommand().SetServerDetails(lcDetails).SetReleaseBundleName(c.Args().Get(0)).
 		SetReleaseBundleVersion(c.Args().Get(1)).SetSigningKeyName(c.String(cliutils.SigningKey)).SetSync(c.Bool(cliutils.Sync)).
-		SetReleaseBundleProject(cliutils.GetProject(c)).SetBuildsSpecPath(c.String(cliutils.Builds)).
-		SetReleaseBundlesSpecPath(c.String(cliutils.ReleaseBundles))
+		SetReleaseBundleProject(cliutils.GetProject(c)).SetSpec(creationSpec).
+		SetBuildsSpecPath(c.String(cliutils.Builds)).SetReleaseBundlesSpecPath(c.String(cliutils.ReleaseBundles))
 	return commands.Exec(createCmd)
 }
 

lifecycle/cli_test.go

@@ -24,8 +24,11 @@ func TestValidateCreateReleaseBundleContext(t *testing.T) {
 		{"builds correct", []string{"name", "version"}, []string{
 			cliutils.Builds + "=/path/to/file", cliutils.SigningKey + "=key"}, false},
 		{"releaseBundles without signing key", []string{"name", "version", "env"}, []string{cliutils.ReleaseBundles + "=/path/to/file"}, true},
-		{"releaseBundles", []string{"name", "version"}, []string{
+		{"releaseBundles correct", []string{"name", "version"}, []string{
 			cliutils.ReleaseBundles + "=/path/to/file", cliutils.SigningKey + "=key"}, false},
+		{"spec without signing key", []string{"name", "version", "env"}, []string{"spec=/path/to/file"}, true},
+		{"spec correct", []string{"name", "version"}, []string{
+			"spec=/path/to/file", cliutils.SigningKey + "=key"}, false},
 	}
 
 	for _, test := range testRuns {

lifecycle_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 	"time"
@@ -41,7 +42,7 @@ var (
 	lcCli     *coreTests.JfrogCli
 )
 
-func TestLifecycle(t *testing.T) {
+func TestBackwardCompatibleReleaseBundleCreation(t *testing.T) {
 	cleanHomeDir := initLifecycleTest(t)
 	defer cleanHomeDir()
 	defer cleanLifecycleTests(t)
@@ -52,17 +53,84 @@ func TestLifecycle(t *testing.T) {
 	defer deleteBuilds()
 
 	// Create release bundle from builds synchronously.
-	createRb(t, buildsSpec12, cliutils.Builds, tests.LcRbName1, number1, true)
+	createRbBackwardCompatible(t, buildsSpec12, cliutils.Builds, tests.LcRbName1, number1, true)
 	defer deleteReleaseBundle(t, lcManager, tests.LcRbName1, number1)
 
 	// Create release bundle from a build asynchronously and assert status.
 	// This build has dependencies which are included in the release bundle.
-	createRb(t, buildsSpec3, cliutils.Builds, tests.LcRbName2, number2, false)
+	createRbBackwardCompatible(t, buildsSpec3, cliutils.Builds, tests.LcRbName2, number2, false)
 	defer deleteReleaseBundle(t, lcManager, tests.LcRbName2, number2)
 	assertStatusCompleted(t, lcManager, tests.LcRbName2, number2, "")
 
 	// Create a combined release bundle from the two previous release bundle.
-	createRb(t, releaseBundlesSpec, cliutils.ReleaseBundles, tests.LcRbName3, number3, true)
+	createRbBackwardCompatible(t, releaseBundlesSpec, cliutils.ReleaseBundles, tests.LcRbName3, number3, true)
+	defer deleteReleaseBundle(t, lcManager, tests.LcRbName3, number3)
+
+	assertRbArtifacts(t, lcManager, tests.LcRbName3, number3, tests.GetExpectedBackwardCompatibleLifecycleArtifacts())
+}
+
+func assertRbArtifacts(t *testing.T, lcManager *lifecycle.LifecycleServicesManager, rbName, rbVersion string, expected []string) {
+	specs, err := getReleaseBundleSpecification(lcManager, rbName, rbVersion)
+	if !assert.NoError(t, err) {
+		return
+	}
+	compareRbArtifacts(t, specs, expected)
+}
+
+func compareRbArtifacts(t *testing.T, actual services.ReleaseBundleSpecResponse, expected []string) {
+	var actualArtifactsPaths []string
+	for _, artifact := range actual.Artifacts {
+		actualArtifactsPaths = append(actualArtifactsPaths, path.Join(artifact.SourceRepositoryKey, artifact.Path))
+	}
+	assert.ElementsMatch(t, actualArtifactsPaths, expected)
+}
+
+func TestReleaseBundleCreationFromAql(t *testing.T) {
+	testReleaseBundleCreation(t, tests.UploadDevSpecA, tests.LifecycleAql, tests.GetExpectedLifecycleCreationByAql())
+}
+
+func TestReleaseBundleCreationFromArtifacts(t *testing.T) {
+	testReleaseBundleCreation(t, tests.UploadDevSpec, tests.LifecycleArtifacts, tests.GetExpectedLifecycleCreationByArtifacts())
+}
+
+func testReleaseBundleCreation(t *testing.T, uploadSpec, creationSpec string, expected []string) {
+	cleanHomeDir := initLifecycleTest(t)
+	defer cleanHomeDir()
+	defer cleanLifecycleTests(t)
+	lcManager := getLcServiceManager(t)
+
+	specFile, err := tests.CreateSpec(uploadSpec)
+	assert.NoError(t, err)
+	runRt(t, "upload", "--spec="+specFile)
+
+	createRbFromSpec(t, creationSpec, tests.LcRbName1, number1, true)
+	defer deleteReleaseBundle(t, lcManager, tests.LcRbName1, number1)
+
+	assertRbArtifacts(t, lcManager, tests.LcRbName1, number1, expected)
+}
+
+func TestLifecycleFullFlow(t *testing.T) {
+	cleanHomeDir := initLifecycleTest(t)
+	defer cleanHomeDir()
+	defer cleanLifecycleTests(t)
+	lcManager := getLcServiceManager(t)
+
+	// Upload builds to create release bundles from.
+	deleteBuilds := uploadBuilds(t)
+	defer deleteBuilds()
+
+	// Create release bundle from builds synchronously.
+	createRbFromSpec(t, tests.LifecycleBuilds12, tests.LcRbName1, number1, true)
+	defer deleteReleaseBundle(t, lcManager, tests.LcRbName1, number1)
+
+	// Create release bundle from a build asynchronously and assert status.
+	// This build has dependencies which are included in the release bundle.
+	createRbFromSpec(t, tests.LifecycleBuilds3, tests.LcRbName2, number2, false)
+	defer deleteReleaseBundle(t, lcManager, tests.LcRbName2, number2)
+	assertStatusCompleted(t, lcManager, tests.LcRbName2, number2, "")
+
+	// Create a combined release bundle from the two previous release bundle.
+	createRbFromSpec(t, tests.LifecycleReleaseBundles, tests.LcRbName3, number3, true)
 	defer deleteReleaseBundle(t, lcManager, tests.LcRbName3, number3)
 
 	// Promote the last release bundle to prod repo 1.
@@ -101,14 +169,24 @@ func uploadBuilds(t *testing.T) func() {
 	}
 }
 
-func createRb(t *testing.T, specName, sourceOption, rbName, rbVersion string, sync bool) {
+func createRbBackwardCompatible(t *testing.T, specName, sourceOption, rbName, rbVersion string, sync bool) {
 	specFile, err := getSpecFile(specName)
 	assert.NoError(t, err)
+	createRb(t, specFile, sourceOption, rbName, rbVersion, sync)
+}
+
+func createRbFromSpec(t *testing.T, specName, rbName, rbVersion string, sync bool) {
+	specFile, err := tests.CreateSpec(specName)
+	assert.NoError(t, err)
+	createRb(t, specFile, "spec", rbName, rbVersion, sync)
+}
+
+func createRb(t *testing.T, specFilePath, sourceOption, rbName, rbVersion string, sync bool) {
 	argsAndOptions := []string{
 		"rbc",
 		rbName,
 		rbVersion,
-		getOption(sourceOption, specFile),
+		getOption(sourceOption, specFilePath),
 		getOption(cliutils.SigningKey, gpgKeyPairName),
 	}
 	// Add the --sync option only if requested, to test the default value.
@@ -198,6 +276,15 @@ func getStatus(lcManager *lifecycle.LifecycleServicesManager, rbName, rbVersion,
 	return lcManager.GetReleaseBundlePromotionStatus(rbDetails, "", createdMillis, true)
 }
 
+func getReleaseBundleSpecification(lcManager *lifecycle.LifecycleServicesManager, rbName, rbVersion string) (services.ReleaseBundleSpecResponse, error) {
+	rbDetails := services.ReleaseBundleDetails{
+		ReleaseBundleName:    rbName,
+		ReleaseBundleVersion: rbVersion,
+	}
+
+	return lcManager.GetReleaseBundleSpecification(rbDetails)
+}
+
 func deleteReleaseBundle(t *testing.T, lcManager *lifecycle.LifecycleServicesManager, rbName, rbVersion string) {
 	rbDetails := services.ReleaseBundleDetails{
 		ReleaseBundleName:    rbName,

schema/filespec-schema.json

@@ -217,6 +217,10 @@
       "gpg-key": {
         "type": "string",
         "description": "Path to the public GPG key file located on the file system, used to validate downloaded release bundles."
+      },
+      "project": {
+        "type": "string",
+        "description": "JFrog Artifactory project key."
       }
     },
 

testdata/filespecs/lifecycle-aql.json

@@ -0,0 +1,13 @@
+{
+  "files": [
+    {
+      "aql": {
+        "items.find": {
+          "repo": "${DEV_REPO}",
+          "path": ".",
+          "name": "a2.in"
+        }
+      }
+    }
+  ]
+}

testdata/filespecs/lifecycle-artifacts.json

@@ -0,0 +1,11 @@
+{
+  "files": [
+    {
+      "pattern": "${DEV_REPO}/a*",
+      "exclusions": ["${DEV_REPO}/a1.in","${DEV_REPO}/a3.in"]
+    },
+    {
+      "pattern": "${DEV_REPO}/b*"
+    }
+  ]
+}

testdata/filespecs/lifecycle-builds-1-2.json

@@ -0,0 +1,11 @@
+{
+  "files": [
+    {
+      "build": "${LC_BUILD_NAME1}/111"
+    },
+    {
+      "build": "${LC_BUILD_NAME2}",
+      "project": "default"
+    }
+  ]
+}

testdata/filespecs/lifecycle-builds-3.json

@@ -0,0 +1,8 @@
+{
+  "files": [
+    {
+      "build": "${LC_BUILD_NAME3}",
+      "includeDeps": "true"
+    }
+  ]
+}

testdata/filespecs/lifecycle-release-bundles.json

@@ -0,0 +1,10 @@
+{
+  "files": [
+    {
+      "bundle": "${RB_NAME1}/111"
+    },
+    {
+      "bundle": "${RB_NAME2}/222"
+    }
+  ]
+}

testdata/filespecs/upload_dev_spec.json

@@ -0,0 +1,10 @@
+{
+  "files": [
+    {
+      "pattern": "testdata/a/*",
+      "target": "${DEV_REPO}/",
+      "flat": "true",
+      "recursive": "true"
+    }
+  ]
+}

utils/cliutils/commandsflags.go

@@ -1627,12 +1627,14 @@ var flagsMap = map[string]cli.Flag{
 		Usage: "[Optional] Project key associated with the Release Bundle version.` `",
 	},
 	lcBuilds: cli.StringFlag{
-		Name:  Builds,
-		Usage: "[Optional] Path to a JSON file containing information of the source builds from which to create a release bundle.` `",
+		Name:   Builds,
+		Usage:  "[Optional] Path to a JSON file containing information of the source builds from which to create a release bundle.` `",
+		Hidden: true,
 	},
 	lcReleaseBundles: cli.StringFlag{
-		Name:  ReleaseBundles,
-		Usage: "[Optional] Path to a JSON file containing information of the source release bundles from which to create a release bundle.` `",
+		Name:   ReleaseBundles,
+		Usage:  "[Optional] Path to a JSON file containing information of the source release bundles from which to create a release bundle.` `",
+		Hidden: true,
 	},
 	lcSigningKey: cli.StringFlag{
 		Name:  SigningKey,
@@ -1992,6 +1994,7 @@ var commandFlags = map[string][]string{
 	},
 	ReleaseBundleCreate: {
 		lcUrl, user, password, accessToken, serverId, lcSigningKey, lcSync, lcProject, lcBuilds, lcReleaseBundles,
+		specFlag, specVars,
 	},
 	ReleaseBundlePromote: {
 		lcUrl, user, password, accessToken, serverId, lcSigningKey, lcSync, lcProject, lcIncludeRepos, lcExcludeRepos,