diff --git a/alpha/action/migrate.go b/alpha/action/migrate.go index c5880f000..3012c2a0e 100644 --- a/alpha/action/migrate.go +++ b/alpha/action/migrate.go @@ -29,14 +29,11 @@ func (m Migrate) Run(ctx context.Context) error { } r := Render{ - Refs: []string{m.CatalogRef}, + Refs: []string{m.CatalogRef}, + Migrate: true, - // Only allow sqlite images and files to be migrated. Other types cannot - // always be migrated cleanly because they may contain file references. - // Rendered sqlite databases never contain file references. - AllowedRefMask: RefSqliteImage | RefSqliteFile, - - skipSqliteDeprecationLog: true, + // Only allow catalogs to be migrated. + AllowedRefMask: RefSqliteImage | RefSqliteFile | RefDCImage | RefDCDir, } if m.Registry != nil { r.Registry = m.Registry diff --git a/alpha/action/migrate_test.go b/alpha/action/migrate_test.go index 5f6be7b95..2e889c8c0 100644 --- a/alpha/action/migrate_test.go +++ b/alpha/action/migrate_test.go @@ -50,8 +50,8 @@ func TestMigrate(t *testing.T) { Registry: reg, }, expectedFiles: map[string]string{ - "foo/catalog.yaml": migrateFooCatalog(), - "bar/catalog.yaml": migrateBarCatalog(), + "foo/catalog.yaml": migrateFooCatalogSqlite(), + "bar/catalog.yaml": migrateBarCatalogSqlite(), }, }, { @@ -64,12 +64,12 @@ func TestMigrate(t *testing.T) { Registry: reg, }, expectedFiles: map[string]string{ - "foo/catalog.yaml": migrateFooCatalog(), - "bar/catalog.yaml": migrateBarCatalog(), + "foo/catalog.yaml": migrateFooCatalogSqlite(), + "bar/catalog.yaml": migrateBarCatalogSqlite(), }, }, { - name: "DeclcfgImage/Failure", + name: "DeclcfgImage/Success", migrate: action.Migrate{ CatalogRef: "test.registry/foo-operator/foo-index-declcfg:v0.2.0", OutputDir: filepath.Join(tmpDir, "declcfg-image"), @@ -77,10 +77,12 @@ func TestMigrate(t *testing.T) { FileExt: ".yaml", Registry: reg, }, - expectErr: action.ErrNotAllowed, + expectedFiles: map[string]string{ + "foo/catalog.yaml": migrateFooCatalogFBC(), + }, }, { - name: "DeclcfgDir/Failure", + name: "DeclcfgDir/Success", migrate: action.Migrate{ CatalogRef: "testdata/foo-index-v0.2.0-declcfg", OutputDir: filepath.Join(tmpDir, "declcfg-dir"), @@ -88,7 +90,9 @@ func TestMigrate(t *testing.T) { FileExt: ".yaml", Registry: reg, }, - expectErr: action.ErrNotAllowed, + expectedFiles: map[string]string{ + "foo/catalog.yaml": migrateFooCatalogFBC(), + }, }, { name: "BundleImage/Failure", @@ -106,12 +110,22 @@ func TestMigrate(t *testing.T) { t.Run(s.name, func(t *testing.T) { err := s.migrate.Run(context.Background()) require.ErrorIs(t, err, s.expectErr) - for file, expectedData := range s.expectedFiles { - path := filepath.Join(s.migrate.OutputDir, file) - actualData, err := os.ReadFile(path) + if s.expectErr != nil { + return + } + actualFS := os.DirFS(s.migrate.OutputDir) + fs.WalkDir(actualFS, ".", func(path string, d fs.DirEntry, err error) error { require.NoError(t, err) + if d.IsDir() { + return nil + } + actualData, err := fs.ReadFile(actualFS, path) + require.NoError(t, err) + expectedData, ok := s.expectedFiles[path] + require.True(t, ok, "output directory contained unexpected file %q", path) require.Equal(t, expectedData, string(actualData)) - } + return nil + }) }) } } @@ -156,7 +170,7 @@ func newMigrateRegistry(t *testing.T, imageMap map[image.Reference]string) (imag return reg, nil } -func migrateFooCatalog() string { +func migrateFooCatalogSqlite() string { return `--- defaultChannel: beta name: foo @@ -280,7 +294,7 @@ schema: olm.bundle ` } -func migrateBarCatalog() string { +func migrateBarCatalogSqlite() string { return `--- defaultChannel: alpha name: bar @@ -357,3 +371,135 @@ relatedImages: schema: olm.bundle ` } + +func migrateFooCatalogFBC() string { + return `--- +defaultChannel: beta +name: foo +properties: +- type: owner + value: + group: abc.com + name: admin +schema: olm.package +--- +entries: +- name: foo.v0.1.0 + skipRange: <0.1.0 +- name: foo.v0.2.0 + replaces: foo.v0.1.0 + skipRange: <0.2.0 + skips: + - foo.v0.1.1 + - foo.v0.1.2 +name: beta +package: foo +properties: +- type: user + value: + group: xyz.com + name: account +schema: olm.channel +--- +entries: +- name: foo.v0.2.0 + replaces: foo.v0.1.0 + skipRange: <0.2.0 + skips: + - foo.v0.1.1 + - foo.v0.1.2 +name: stable +package: foo +schema: olm.channel +--- +image: test.registry/foo-operator/foo-bundle:v0.1.0 +name: foo.v0.1.0 +package: foo +properties: +- type: olm.gvk + value: + group: test.foo + kind: Foo + version: v1 +- type: olm.gvk.required + value: + group: test.bar + kind: Bar + version: v1alpha1 +- type: olm.package + value: + packageName: foo + version: 0.1.0 +- type: olm.package.required + value: + packageName: bar + versionRange: <0.1.0 +- type: olm.csv.metadata + value: + annotations: + olm.skipRange: <0.1.0 + apiServiceDefinitions: {} + crdDescriptions: + owned: + - kind: Foo + name: foos.test.foo + version: v1 + displayName: Foo Operator + provider: {} +relatedImages: +- image: test.registry/foo-operator/foo-bundle:v0.1.0 + name: "" +- image: test.registry/foo-operator/foo:v0.1.0 + name: operator +schema: olm.bundle +--- +image: test.registry/foo-operator/foo-bundle:v0.2.0 +name: foo.v0.2.0 +package: foo +properties: +- type: olm.gvk + value: + group: test.foo + kind: Foo + version: v1 +- type: olm.gvk.required + value: + group: test.bar + kind: Bar + version: v1alpha1 +- type: olm.package + value: + packageName: foo + version: 0.2.0 +- type: olm.package.required + value: + packageName: bar + versionRange: <0.1.0 +- type: olm.csv.metadata + value: + annotations: + olm.skipRange: <0.2.0 + apiServiceDefinitions: {} + crdDescriptions: + owned: + - kind: Foo + name: foos.test.foo + version: v1 + displayName: Foo Operator + provider: {} +relatedImages: +- image: test.registry/foo-operator/foo-2:v0.2.0 + name: "" +- image: test.registry/foo-operator/foo-bundle:v0.2.0 + name: "" +- image: test.registry/foo-operator/foo-init-2:v0.2.0 + name: "" +- image: test.registry/foo-operator/foo-init:v0.2.0 + name: "" +- image: test.registry/foo-operator/foo-other:v0.2.0 + name: other +- image: test.registry/foo-operator/foo:v0.2.0 + name: operator +schema: olm.bundle +` +} diff --git a/alpha/action/render.go b/alpha/action/render.go index b1e5bd754..f6ed285b4 100644 --- a/alpha/action/render.go +++ b/alpha/action/render.go @@ -15,6 +15,7 @@ import ( "github.com/h2non/filetype" "github.com/h2non/filetype/matchers" + "github.com/operator-framework/api/pkg/operators/v1alpha1" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" @@ -52,6 +53,7 @@ type Render struct { Refs []string Registry image.Registry AllowedRefMask RefType + Migrate bool skipSqliteDeprecationLog bool } @@ -90,6 +92,12 @@ func (r Render) Run(ctx context.Context) (*declcfg.DeclarativeConfig, error) { }) } + if r.Migrate { + if err := migrate(cfg); err != nil { + return nil, fmt.Errorf("migrate: %v", err) + } + } + cfgs = append(cfgs, *cfg) } @@ -395,6 +403,50 @@ func moveBundleObjectsToEndOfPropertySlices(cfg *declcfg.DeclarativeConfig) { } } +func migrate(cfg *declcfg.DeclarativeConfig) error { + migrations := []func(*declcfg.DeclarativeConfig) error{ + convertObjectsToCSVMetadata, + } + + for _, m := range migrations { + if err := m(cfg); err != nil { + return err + } + } + return nil +} + +func convertObjectsToCSVMetadata(cfg *declcfg.DeclarativeConfig) error { +BundleLoop: + for bi, b := range cfg.Bundles { + if b.Image == "" || b.CsvJSON == "" { + continue + } + + var csv v1alpha1.ClusterServiceVersion + if err := json.Unmarshal([]byte(b.CsvJSON), &csv); err != nil { + return err + } + + props := b.Properties[:0] + for _, p := range b.Properties { + switch p.Type { + case property.TypeBundleObject: + // Get rid of the bundle objects + case property.TypeCSVMetadata: + // If this bundle already has a CSV metadata + // property, we won't mutate the bundle at all. + continue BundleLoop + default: + // Keep all of the other properties + props = append(props, p) + } + } + cfg.Bundles[bi].Properties = append(props, property.MustBuildCSVMetadata(csv)) + } + return nil +} + func combineConfigs(cfgs []declcfg.DeclarativeConfig) *declcfg.DeclarativeConfig { out := &declcfg.DeclarativeConfig{} for _, in := range cfgs { diff --git a/alpha/action/render_test.go b/alpha/action/render_test.go index 976da6922..5c2043825 100644 --- a/alpha/action/render_test.go +++ b/alpha/action/render_test.go @@ -294,8 +294,8 @@ func TestRender(t *testing.T) { property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), property.MustBuildPackage("foo", "0.1.0"), property.MustBuildPackageRequired("bar", "<0.1.0"), - property.MustBuildBundleObjectData(foov1csv), - property.MustBuildBundleObjectData(foov1crd), + property.MustBuildBundleObject(foov1csv), + property.MustBuildBundleObject(foov1crd), }, RelatedImages: []declcfg.RelatedImage{ { @@ -319,8 +319,8 @@ func TestRender(t *testing.T) { property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), property.MustBuildPackage("foo", "0.2.0"), property.MustBuildPackageRequired("bar", "<0.1.0"), - property.MustBuildBundleObjectData(foov2csv), - property.MustBuildBundleObjectData(foov2crd), + property.MustBuildBundleObject(foov2csv), + property.MustBuildBundleObject(foov2crd), }, RelatedImages: []declcfg.RelatedImage{ { @@ -392,8 +392,8 @@ func TestRender(t *testing.T) { property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), property.MustBuildPackage("foo", "0.1.0"), property.MustBuildPackageRequired("bar", "<0.1.0"), - property.MustBuildBundleObjectData(foov1csv), - property.MustBuildBundleObjectData(foov1crd), + property.MustBuildBundleObject(foov1csv), + property.MustBuildBundleObject(foov1crd), }, RelatedImages: []declcfg.RelatedImage{ { @@ -417,8 +417,202 @@ func TestRender(t *testing.T) { property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), property.MustBuildPackage("foo", "0.2.0"), property.MustBuildPackageRequired("bar", "<0.1.0"), - property.MustBuildBundleObjectData(foov2csv), - property.MustBuildBundleObjectData(foov2crd), + property.MustBuildBundleObject(foov2csv), + property.MustBuildBundleObject(foov2crd), + }, + RelatedImages: []declcfg.RelatedImage{ + { + Image: "test.registry/foo-operator/foo-2:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-bundle:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-init-2:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-init:v0.2.0", + }, + { + Name: "other", + Image: "test.registry/foo-operator/foo-other:v0.2.0", + }, + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.2.0", + }, + }, + CsvJSON: string(foov2csv), + Objects: []string{string(foov2csv), string(foov2crd)}, + }, + }, + }, + assertion: require.NoError, + }, + { + name: "Success/DeclcfgImageMigrate", + render: action.Render{ + Refs: []string{"test.registry/foo-operator/foo-index-declcfg:v0.2.0"}, + Migrate: true, + Registry: reg, + }, + expectCfg: &declcfg.DeclarativeConfig{ + Packages: []declcfg.Package{ + { + Schema: "olm.package", + Name: "foo", + DefaultChannel: "beta", + Properties: []property.Property{ + {Type: "owner", Value: json.RawMessage("{\"group\":\"abc.com\",\"name\":\"admin\"}")}, + }, + }, + }, + Channels: []declcfg.Channel{ + {Schema: "olm.channel", Package: "foo", Name: "beta", Entries: []declcfg.ChannelEntry{ + {Name: "foo.v0.1.0", SkipRange: "<0.1.0"}, + {Name: "foo.v0.2.0", Replaces: "foo.v0.1.0", SkipRange: "<0.2.0", Skips: []string{"foo.v0.1.1", "foo.v0.1.2"}}, + }, + Properties: []property.Property{ + {Type: "user", Value: json.RawMessage("{\"group\":\"xyz.com\",\"name\":\"account\"}")}, + }, + }, + {Schema: "olm.channel", Package: "foo", Name: "stable", Entries: []declcfg.ChannelEntry{ + {Name: "foo.v0.2.0", Replaces: "foo.v0.1.0", SkipRange: "<0.2.0", Skips: []string{"foo.v0.1.1", "foo.v0.1.2"}}, + }}, + }, + Bundles: []declcfg.Bundle{ + { + Schema: "olm.bundle", + Name: "foo.v0.1.0", + Package: "foo", + Image: "test.registry/foo-operator/foo-bundle:v0.1.0", + Properties: []property.Property{ + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), + property.MustBuildPackage("foo", "0.1.0"), + property.MustBuildPackageRequired("bar", "<0.1.0"), + mustBuildCSVMetadata(bytes.NewReader(foov1csv)), + }, + RelatedImages: []declcfg.RelatedImage{ + { + Image: "test.registry/foo-operator/foo-bundle:v0.1.0", + }, + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.1.0", + }, + }, + CsvJSON: string(foov1csv), + Objects: []string{string(foov1csv), string(foov1crd)}, + }, + { + Schema: "olm.bundle", + Name: "foo.v0.2.0", + Package: "foo", + Image: "test.registry/foo-operator/foo-bundle:v0.2.0", + Properties: []property.Property{ + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), + property.MustBuildPackage("foo", "0.2.0"), + property.MustBuildPackageRequired("bar", "<0.1.0"), + mustBuildCSVMetadata(bytes.NewReader(foov2csv)), + }, + RelatedImages: []declcfg.RelatedImage{ + { + Image: "test.registry/foo-operator/foo-2:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-bundle:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-init-2:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-init:v0.2.0", + }, + { + Name: "other", + Image: "test.registry/foo-operator/foo-other:v0.2.0", + }, + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.2.0", + }, + }, + CsvJSON: string(foov2csv), + Objects: []string{string(foov2csv), string(foov2crd)}, + }, + }, + }, + assertion: require.NoError, + }, + { + name: "Success/DeclcfgDirectoryMigrate", + render: action.Render{ + Refs: []string{"testdata/foo-index-v0.2.0-declcfg"}, + Migrate: true, + Registry: reg, + }, + expectCfg: &declcfg.DeclarativeConfig{ + Packages: []declcfg.Package{ + { + Schema: "olm.package", + Name: "foo", + DefaultChannel: "beta", + Properties: []property.Property{ + {Type: "owner", Value: json.RawMessage("{\"group\":\"abc.com\",\"name\":\"admin\"}")}, + }, + }, + }, + Channels: []declcfg.Channel{ + {Schema: "olm.channel", Package: "foo", Name: "beta", Entries: []declcfg.ChannelEntry{ + {Name: "foo.v0.1.0", SkipRange: "<0.1.0"}, + {Name: "foo.v0.2.0", Replaces: "foo.v0.1.0", SkipRange: "<0.2.0", Skips: []string{"foo.v0.1.1", "foo.v0.1.2"}}, + }, + Properties: []property.Property{ + {Type: "user", Value: json.RawMessage("{\"group\":\"xyz.com\",\"name\":\"account\"}")}, + }, + }, + {Schema: "olm.channel", Package: "foo", Name: "stable", Entries: []declcfg.ChannelEntry{ + {Name: "foo.v0.2.0", Replaces: "foo.v0.1.0", SkipRange: "<0.2.0", Skips: []string{"foo.v0.1.1", "foo.v0.1.2"}}, + }}, + }, + Bundles: []declcfg.Bundle{ + { + Schema: "olm.bundle", + Name: "foo.v0.1.0", + Package: "foo", + Image: "test.registry/foo-operator/foo-bundle:v0.1.0", + Properties: []property.Property{ + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), + property.MustBuildPackage("foo", "0.1.0"), + property.MustBuildPackageRequired("bar", "<0.1.0"), + mustBuildCSVMetadata(bytes.NewReader(foov1csv)), + }, + RelatedImages: []declcfg.RelatedImage{ + { + Image: "test.registry/foo-operator/foo-bundle:v0.1.0", + }, + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.1.0", + }, + }, + CsvJSON: string(foov1csv), + Objects: []string{string(foov1csv), string(foov1crd)}, + }, + { + Schema: "olm.bundle", + Name: "foo.v0.2.0", + Package: "foo", + Image: "test.registry/foo-operator/foo-bundle:v0.2.0", + Properties: []property.Property{ + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), + property.MustBuildPackage("foo", "0.2.0"), + property.MustBuildPackageRequired("bar", "<0.1.0"), + mustBuildCSVMetadata(bytes.NewReader(foov2csv)), }, RelatedImages: []declcfg.RelatedImage{ { diff --git a/alpha/declcfg/helpers_test.go b/alpha/declcfg/helpers_test.go index cfeb41983..719ba34d8 100644 --- a/alpha/declcfg/helpers_test.go +++ b/alpha/declcfg/helpers_test.go @@ -3,7 +3,6 @@ package declcfg import ( "encoding/json" "fmt" - "path/filepath" "sort" "testing" @@ -116,8 +115,8 @@ func newTestBundle(packageName, version string, opts ...bundleOpt) Bundle { Image: testBundleImage(packageName, version), Properties: []property.Property{ property.MustBuildPackage(packageName, version), - property.MustBuildBundleObjectRef(filepath.Join("objects", testBundleName(packageName, version)+".csv.yaml")), - property.MustBuildBundleObjectData([]byte(`{"kind": "CustomResourceDefinition", "apiVersion": "apiextensions.k8s.io/v1"}`)), + property.MustBuildBundleObject([]byte(csvJson)), + property.MustBuildBundleObject([]byte(`{"kind": "CustomResourceDefinition", "apiVersion": "apiextensions.k8s.io/v1"}`)), }, RelatedImages: []RelatedImage{ { @@ -193,8 +192,8 @@ func getBundle(pkg *model.Package, ch *model.Channel, version, replaces string, Image: testBundleImage(pkg.Name, version), Properties: []property.Property{ property.MustBuildPackage(pkg.Name, version), - property.MustBuildBundleObjectRef(filepath.Join("objects", testBundleName(pkg.Name, version)+".csv.yaml")), - property.MustBuildBundleObjectData([]byte(getCRDJSON())), + property.MustBuildBundleObject([]byte(getCSVJson(pkg.Name, version))), + property.MustBuildBundleObject([]byte(getCRDJSON())), }, Replaces: replaces, Skips: skips, diff --git a/alpha/declcfg/load.go b/alpha/declcfg/load.go index 43864f792..40528d902 100644 --- a/alpha/declcfg/load.go +++ b/alpha/declcfg/load.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "io/fs" - "path/filepath" "runtime" "sync" @@ -237,25 +236,19 @@ func mergeCfgs(ctx context.Context, cfgChan <-chan *DeclarativeConfig, fcfg *Dec } } -func readBundleObjects(bundles []Bundle, root fs.FS, path string) error { +func readBundleObjects(bundles []Bundle) error { for bi, b := range bundles { - props, err := property.Parse(b.Properties) - if err != nil { - return fmt.Errorf("package %q, bundle %q: parse properties: %v", b.Package, b.Name, err) - } - for oi, obj := range props.BundleObjects { - objID := fmt.Sprintf(" %q", obj.GetRef()) - if !obj.IsRef() { - objID = fmt.Sprintf("[%d]", oi) + var obj property.BundleObject + for i, props := range b.Properties { + if props.Type != property.TypeBundleObject { + continue } - - d, err := obj.GetData(root, filepath.Dir(path)) - if err != nil { - return fmt.Errorf("package %q, bundle %q: get data for bundle object%s: %v", b.Package, b.Name, objID, err) + if err := json.Unmarshal(props.Value, &obj); err != nil { + return fmt.Errorf("package %q, bundle %q: parse property at index %d as bundle object: %v", b.Package, b.Name, i, err) } - objJson, err := yaml.ToJSON(d) + objJson, err := yaml.ToJSON(obj.Data) if err != nil { - return fmt.Errorf("package %q, bundle %q: convert object%s to JSON: %v", b.Package, b.Name, objID, err) + return fmt.Errorf("package %q, bundle %q: convert bundle object property at index %d to JSON: %v", b.Package, b.Name, i, err) } bundles[bi].Objects = append(bundles[bi].Objects, string(objJson)) } @@ -278,7 +271,6 @@ func extractCSV(objs []string) string { } // LoadReader reads yaml or json from the passed in io.Reader and unmarshals it into a DeclarativeConfig struct. -// Path references will not be de-referenced so callers are responsible for de-referencing if necessary. func LoadReader(r io.Reader) (*DeclarativeConfig, error) { cfg := &DeclarativeConfig{} @@ -314,6 +306,11 @@ func LoadReader(r io.Reader) (*DeclarativeConfig, error) { }); err != nil { return nil, err } + + if err := readBundleObjects(cfg.Bundles); err != nil { + return nil, fmt.Errorf("read bundle objects: %v", err) + } + return cfg, nil } @@ -331,9 +328,5 @@ func LoadFile(root fs.FS, path string) (*DeclarativeConfig, error) { return nil, err } - if err := readBundleObjects(cfg.Bundles, root, path); err != nil { - return nil, fmt.Errorf("read bundle objects: %v", err) - } - return cfg, nil } diff --git a/alpha/declcfg/load_test.go b/alpha/declcfg/load_test.go index c3f769c9b..828beaf7e 100644 --- a/alpha/declcfg/load_test.go +++ b/alpha/declcfg/load_test.go @@ -2,7 +2,9 @@ package declcfg import ( "context" + "encoding/base64" "encoding/json" + "fmt" "io/fs" "os" "testing" @@ -254,7 +256,7 @@ func TestLoadFS(t *testing.T) { {Type: "olm.gvk", Value: json.RawMessage(`{"group":"etcd.database.coreos.com","kind":"EtcdCluster","version":"v1beta2"}`)}, {Type: "olm.channel", Value: json.RawMessage(`{"name":"alpha"}`)}, {Type: "olm.skipRange", Value: json.RawMessage(`"<0.6.1"`)}, - {Type: "olm.bundle.object", Value: json.RawMessage(`{"ref":"etcdoperator.v0.6.1.clusterserviceversion.yaml"}`)}, + {Type: "olm.bundle.object", Value: json.RawMessage(fmt.Sprintf(`{"data": %q}`, base64.StdEncoding.EncodeToString(etcdCSV.Data)))}, }, RelatedImages: []RelatedImage{{Name: "etcdv0.6.1", Image: "quay.io/coreos/etcd-operator@sha256:bd944a211eaf8f31da5e6d69e8541e7cada8f16a9f7a5a570b22478997819943"}}, Objects: []string{toJSON(t, etcdCSV.Data)}, @@ -508,7 +510,7 @@ var ( }`), } etcd = &fstest.MapFile{ - Data: []byte(`--- + Data: []byte(fmt.Sprintf(`--- schema: olm.package name: etcd defaultChannel: singlenamespace-alpha @@ -539,7 +541,7 @@ properties: value: <0.6.1 - type: olm.bundle.object value: - ref: etcdoperator.v0.6.1.clusterserviceversion.yaml + data: %q relatedImages: - image: quay.io/coreos/etcd-operator@sha256:bd944a211eaf8f31da5e6d69e8541e7cada8f16a9f7a5a570b22478997819943 name: etcdv0.6.1 @@ -674,7 +676,7 @@ properties: replaces: etcdoperator.v0.9.2-clusterwide relatedImages: - image: quay.io/coreos/etcd-operator@sha256:66a37fd61a06a43969854ee6d3e21087a98b93838e284a6086b13917f96b0d9b - name: etcdv0.9.2`), + name: etcdv0.9.2`, base64.StdEncoding.EncodeToString(etcdCSV.Data))), } etcdCSV = &fstest.MapFile{ Data: []byte(`apiVersion: operators.coreos.com/v1alpha1 diff --git a/alpha/declcfg/write_test.go b/alpha/declcfg/write_test.go index bbbd3482b..19ea075f9 100644 --- a/alpha/declcfg/write_test.go +++ b/alpha/declcfg/write_test.go @@ -78,7 +78,7 @@ func TestWriteJSON(t *testing.T) { { "type": "olm.bundle.object", "value": { - "ref": "objects/anakin.v0.0.1.csv.yaml" + "data": "eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJhbmFraW4udjAuMC4xIn19" } }, { @@ -111,7 +111,7 @@ func TestWriteJSON(t *testing.T) { { "type": "olm.bundle.object", "value": { - "ref": "objects/anakin.v0.1.0.csv.yaml" + "data": "eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJhbmFraW4udjAuMS4wIn19" } }, { @@ -144,7 +144,7 @@ func TestWriteJSON(t *testing.T) { { "type": "olm.bundle.object", "value": { - "ref": "objects/anakin.v0.1.1.csv.yaml" + "data": "eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJhbmFraW4udjAuMS4xIn19" } }, { @@ -206,7 +206,7 @@ func TestWriteJSON(t *testing.T) { { "type": "olm.bundle.object", "value": { - "ref": "objects/boba-fett.v1.0.0.csv.yaml" + "data": "eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJib2JhLWZldHQudjEuMC4wIn19" } }, { @@ -239,7 +239,7 @@ func TestWriteJSON(t *testing.T) { { "type": "olm.bundle.object", "value": { - "ref": "objects/boba-fett.v2.0.0.csv.yaml" + "data": "eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJib2JhLWZldHQudjIuMC4wIn19" } }, { @@ -329,7 +329,7 @@ properties: data: eyJraW5kIjogIkN1c3RvbVJlc291cmNlRGVmaW5pdGlvbiIsICJhcGlWZXJzaW9uIjogImFwaWV4dGVuc2lvbnMuazhzLmlvL3YxIn0= - type: olm.bundle.object value: - ref: objects/anakin.v0.0.1.csv.yaml + data: eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJhbmFraW4udjAuMC4xIn19 - type: olm.package value: packageName: anakin @@ -348,7 +348,7 @@ properties: data: eyJraW5kIjogIkN1c3RvbVJlc291cmNlRGVmaW5pdGlvbiIsICJhcGlWZXJzaW9uIjogImFwaWV4dGVuc2lvbnMuazhzLmlvL3YxIn0= - type: olm.bundle.object value: - ref: objects/anakin.v0.1.0.csv.yaml + data: eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJhbmFraW4udjAuMS4wIn19 - type: olm.package value: packageName: anakin @@ -367,7 +367,7 @@ properties: data: eyJraW5kIjogIkN1c3RvbVJlc291cmNlRGVmaW5pdGlvbiIsICJhcGlWZXJzaW9uIjogImFwaWV4dGVuc2lvbnMuazhzLmlvL3YxIn0= - type: olm.bundle.object value: - ref: objects/anakin.v0.1.1.csv.yaml + data: eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJhbmFraW4udjAuMS4xIn19 - type: olm.package value: packageName: anakin @@ -406,7 +406,7 @@ properties: data: eyJraW5kIjogIkN1c3RvbVJlc291cmNlRGVmaW5pdGlvbiIsICJhcGlWZXJzaW9uIjogImFwaWV4dGVuc2lvbnMuazhzLmlvL3YxIn0= - type: olm.bundle.object value: - ref: objects/boba-fett.v1.0.0.csv.yaml + data: eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJib2JhLWZldHQudjEuMC4wIn19 - type: olm.package value: packageName: boba-fett @@ -425,7 +425,7 @@ properties: data: eyJraW5kIjogIkN1c3RvbVJlc291cmNlRGVmaW5pdGlvbiIsICJhcGlWZXJzaW9uIjogImFwaWV4dGVuc2lvbnMuazhzLmlvL3YxIn0= - type: olm.bundle.object value: - ref: objects/boba-fett.v2.0.0.csv.yaml + data: eyJraW5kIjogIkNsdXN0ZXJTZXJ2aWNlVmVyc2lvbiIsICJhcGlWZXJzaW9uIjogIm9wZXJhdG9ycy5jb3Jlb3MuY29tL3YxYWxwaGExIiwgIm1ldGFkYXRhIjp7Im5hbWUiOiJib2JhLWZldHQudjIuMC4wIn19 - type: olm.package value: packageName: boba-fett diff --git a/alpha/model/model_test.go b/alpha/model/model_test.go index c286dcdb4..029946adb 100644 --- a/alpha/model/model_test.go +++ b/alpha/model/model_test.go @@ -478,7 +478,7 @@ func TestValidators(t *testing.T) { Properties: []property.Property{ property.MustBuildPackage("anakin", "0.1.0"), property.MustBuildGVK("skywalker.me", "v1alpha1", "PodRacer"), - property.MustBuildBundleObjectRef("path/to/data"), + property.MustBuildBundleObject([]byte("testdata")), }, Objects: []string{"testdata"}, CsvJSON: "CSVjson", diff --git a/alpha/property/property.go b/alpha/property/property.go index 5cb876846..ff6c15431 100644 --- a/alpha/property/property.go +++ b/alpha/property/property.go @@ -5,9 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "io/fs" - "io/ioutil" - "path/filepath" "reflect" "github.com/operator-framework/api/pkg/operators/v1alpha1" @@ -70,7 +67,7 @@ type GVKRequired struct { } type BundleObject struct { - File `json:",inline"` + Data []byte `json:"data"` } type CSVMetadata struct { @@ -90,58 +87,6 @@ type CSVMetadata struct { Provider v1alpha1.AppLink `json:"provider,omitempty"` } -type File struct { - ref string - data []byte -} - -type fileJSON struct { - Ref string `json:"ref,omitempty"` - Data []byte `json:"data,omitempty"` -} - -func (f *File) UnmarshalJSON(data []byte) error { - var t fileJSON - if err := json.Unmarshal(data, &t); err != nil { - return err - } - if len(t.Ref) > 0 && len(t.Data) > 0 { - return errors.New("fields 'ref' and 'data' are mutually exclusive") - } - f.ref = t.Ref - f.data = t.Data - return nil -} - -func (f File) MarshalJSON() ([]byte, error) { - return json.Marshal(fileJSON{ - Ref: f.ref, - Data: f.data, - }) -} - -func (f File) IsRef() bool { - return len(f.ref) > 0 -} - -func (f File) GetRef() string { - return f.ref -} - -func (f File) GetData(root fs.FS, cwd string) ([]byte, error) { - if !f.IsRef() { - return f.data, nil - } - if filepath.IsAbs(f.ref) { - return nil, fmt.Errorf("reference must be a relative path") - } - file, err := root.Open(filepath.Join(cwd, f.ref)) - if err != nil { - return nil, err - } - return ioutil.ReadAll(file) -} - type Properties struct { Packages []Package `hash:"set"` PackagesRequired []PackageRequired `hash:"set"` @@ -308,11 +253,8 @@ func MustBuildGVK(group, version, kind string) Property { func MustBuildGVKRequired(group, version, kind string) Property { return MustBuild(&GVKRequired{group, kind, version}) } -func MustBuildBundleObjectRef(ref string) Property { - return MustBuild(&BundleObject{File: File{ref: ref}}) -} -func MustBuildBundleObjectData(data []byte) Property { - return MustBuild(&BundleObject{File: File{data: data}}) +func MustBuildBundleObject(data []byte) Property { + return MustBuild(&BundleObject{Data: data}) } func MustBuildCSVMetadata(csv v1alpha1.ClusterServiceVersion) Property { diff --git a/alpha/property/property_test.go b/alpha/property/property_test.go index 8a05c450b..171cec7a0 100644 --- a/alpha/property/property_test.go +++ b/alpha/property/property_test.go @@ -1,12 +1,7 @@ package property import ( - "encoding/base64" "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -70,158 +65,6 @@ func TestValidate(t *testing.T) { } } -func TestFile_MarshalJSON(t *testing.T) { - type spec struct { - name string - file File - json string - assertion require.ErrorAssertionFunc - } - specs := []spec{ - { - name: "Success/Ref", - file: File{ref: "foo"}, - json: `{"ref":"foo"}`, - assertion: require.NoError, - }, - { - name: "Success/Data", - file: File{data: []byte("foo")}, - json: fmt.Sprintf(`{"data":%q}`, base64.StdEncoding.EncodeToString([]byte("foo"))), - assertion: require.NoError, - }, - } - - for _, s := range specs { - t.Run(s.name, func(t *testing.T) { - d, err := json.Marshal(s.file) - s.assertion(t, err) - assert.Equal(t, s.json, string(d)) - }) - } -} - -func TestFile_UnmarshalJSON(t *testing.T) { - type spec struct { - name string - file File - json string - assertion require.ErrorAssertionFunc - } - specs := []spec{ - { - name: "Success/Ref", - file: File{ref: "foo"}, - json: `{"ref":"foo"}`, - assertion: require.NoError, - }, - { - name: "Success/Data", - file: File{data: []byte("foo")}, - json: fmt.Sprintf(`{"data":%q}`, base64.StdEncoding.EncodeToString([]byte("foo"))), - assertion: require.NoError, - }, - { - name: "Error/RefAndData", - json: fmt.Sprintf(`{"ref":"foo","data":%q}`, base64.StdEncoding.EncodeToString([]byte("bar"))), - assertion: require.Error, - }, - { - name: "Error/InvalidJSON", - json: `["ref","data"]`, - assertion: require.Error, - }, - } - - for _, s := range specs { - t.Run(s.name, func(t *testing.T) { - var actual File - err := json.Unmarshal([]byte(s.json), &actual) - s.assertion(t, err) - assert.Equal(t, s.file, actual) - }) - } -} - -func TestFile_IsRef(t *testing.T) { - assert.True(t, File{ref: "foo"}.IsRef()) - assert.False(t, File{data: []byte("bar")}.IsRef()) -} - -func TestFile_GetRef(t *testing.T) { - assert.Equal(t, "foo", File{ref: "foo"}.GetRef()) - assert.Equal(t, "", File{data: []byte("bar")}.GetRef()) -} - -func TestFile_GetData(t *testing.T) { - type spec struct { - name string - createFile func(root string) error - file File - assertion assert.ErrorAssertionFunc - expectData []byte - } - - createFile := func(root string) error { - dir := filepath.Join(root, "tmp") - if err := os.MkdirAll(dir, 0777); err != nil { - return err - } - return ioutil.WriteFile(filepath.Join(dir, "foo.txt"), []byte("bar"), 0666) - } - - specs := []spec{ - { - name: "Success/NilData", - file: File{}, - assertion: assert.NoError, - expectData: nil, - }, - { - name: "Success/WithData", - file: File{data: []byte("bar")}, - assertion: assert.NoError, - expectData: []byte("bar"), - }, - { - name: "Success/WithRef", - createFile: createFile, - file: File{ref: "tmp/foo.txt"}, - assertion: assert.NoError, - expectData: []byte("bar"), - }, - { - name: "Error/WithRef/FileDoesNotExist", - file: File{ref: "non-existent.txt"}, - assertion: assert.Error, - }, - { - name: "Error/WithRef/RefIsAbsolutePath", - file: File{ref: "/etc/hosts"}, - assertion: assert.Error, - }, - { - name: "Error/WithRef/RefIsOutsideRoot", - file: File{ref: "../etc/hosts"}, - assertion: assert.Error, - }, - } - - for _, s := range specs { - t.Run(s.name, func(t *testing.T) { - dir := t.TempDir() - - if s.createFile != nil { - require.NoError(t, s.createFile(dir)) - } - - data, err := s.file.GetData(os.DirFS(dir), ".") - s.assertion(t, err) - assert.Equal(t, s.expectData, data) - }) - } -} - func TestParse(t *testing.T) { type spec struct { name string @@ -283,8 +126,7 @@ func TestParse(t *testing.T) { MustBuildGVK("group", "v1", "Kind2"), MustBuildGVKRequired("other", "v2", "Kind3"), MustBuildGVKRequired("other", "v2", "Kind4"), - MustBuildBundleObjectRef("testref1"), - MustBuildBundleObjectData([]byte("testdata2")), + MustBuildBundleObject([]byte("testdata2")), {Type: "otherType1", Value: json.RawMessage(`{"v":"otherValue1"}`)}, {Type: "otherType2", Value: json.RawMessage(`["otherValue2"]`)}, }, @@ -306,8 +148,7 @@ func TestParse(t *testing.T) { {"other", "Kind4", "v2"}, }, BundleObjects: []BundleObject{ - {File: File{ref: "testref1"}}, - {File: File{data: []byte("testdata2")}}, + {Data: []byte("testdata2")}, }, Others: []Property{ {Type: "otherType1", Value: json.RawMessage(`{"v":"otherValue1"}`)}, @@ -389,9 +230,9 @@ func TestBuild(t *testing.T) { }, { name: "Success/BundleObject", - input: &BundleObject{File: File{ref: "test"}}, + input: &BundleObject{Data: []byte("test")}, assertion: require.NoError, - expectedProperty: propPtr(MustBuildBundleObjectRef("test")), + expectedProperty: propPtr(MustBuildBundleObject([]byte("test"))), }, { name: "Success/Property", diff --git a/cmd/opm/migrate/cmd.go b/cmd/opm/migrate/cmd.go index 36c73abc7..ca3126da2 100644 --- a/cmd/opm/migrate/cmd.go +++ b/cmd/opm/migrate/cmd.go @@ -27,9 +27,6 @@ parsers that assume that a file contains exactly one valid JSON object. ` + sqlite.DeprecationMessage, Args: cobra.ExactArgs(2), - PersistentPreRun: func(_ *cobra.Command, _ []string) { - sqlite.LogSqliteDeprecation() - }, RunE: func(cmd *cobra.Command, args []string) error { migrate.CatalogRef = args[0] migrate.OutputDir = args[1] diff --git a/cmd/opm/render/cmd.go b/cmd/opm/render/cmd.go index 03cdbcbd3..75be06189 100644 --- a/cmd/opm/render/cmd.go +++ b/cmd/opm/render/cmd.go @@ -66,6 +66,7 @@ database files. }, } cmd.Flags().StringVarP(&output, "output", "o", "json", "Output format of the streamed file-based catalog objects (json|yaml)") + cmd.Flags().BoolVar(&render.Migrate, "migrate", false, "Perform migrations on the rendered FBC") return cmd } diff --git a/pkg/api/api_to_model.go b/pkg/api/api_to_model.go index 06645f7c6..e9408c455 100644 --- a/pkg/api/api_to_model.go +++ b/pkg/api/api_to_model.go @@ -117,7 +117,7 @@ func convertAPIBundleToModelProperties(b *Bundle) ([]property.Property, error) { out = append(out, property.MustBuildCSVMetadata(csv)) } else { for _, obj := range b.Object { - out = append(out, property.MustBuildBundleObjectData([]byte(obj))) + out = append(out, property.MustBuildBundleObject([]byte(obj))) } } diff --git a/pkg/registry/registry_to_model.go b/pkg/registry/registry_to_model.go index 869b8a5c7..431c4d1e0 100644 --- a/pkg/registry/registry_to_model.go +++ b/pkg/registry/registry_to_model.go @@ -108,11 +108,11 @@ func ObjectsAndPropertiesFromBundle(b *Bundle) ([]string, []property.Property, e // Otherwise, make a olm.csv.metadata property if the object is a CSV // (and fallback to olm.bundle.object if parsing the CSV fails). if b.BundleImage == "" { - props = append(props, property.MustBuildBundleObjectData(objData)) + props = append(props, property.MustBuildBundleObject(objData)) } else if obj.GetKind() == operators.ClusterServiceVersionKind { var csv v1alpha1.ClusterServiceVersion if err := json.Unmarshal(objData, &csv); err != nil { - props = append(props, property.MustBuildBundleObjectData(objData)) + props = append(props, property.MustBuildBundleObject(objData)) } else { props = append(props, property.MustBuildCSVMetadata(csv)) }