From a23e82f7c421e2bbd926fafe6a4a25e0b72a402a Mon Sep 17 00:00:00 2001 From: Joe Lanford Date: Thu, 12 Dec 2024 17:02:54 -0500 Subject: [PATCH] wip: olm.package.v2, olm.bundle.v2, olm.package.icon Signed-off-by: Joe Lanford --- alpha/action/migrations/001_v2.go | 140 ++++++++++++++++++++++++ alpha/action/migrations/migrations.go | 1 + alpha/declcfg/declcfg.go | 57 +++++++++- alpha/declcfg/write.go | 147 +++++++++++++------------- 4 files changed, 265 insertions(+), 80 deletions(-) create mode 100644 alpha/action/migrations/001_v2.go diff --git a/alpha/action/migrations/001_v2.go b/alpha/action/migrations/001_v2.go new file mode 100644 index 000000000..fad2def3f --- /dev/null +++ b/alpha/action/migrations/001_v2.go @@ -0,0 +1,140 @@ +package migrations + +import ( + "encoding/json" + "fmt" + + "github.com/operator-framework/operator-registry/alpha/declcfg" + "github.com/operator-framework/operator-registry/alpha/model" + "github.com/operator-framework/operator-registry/alpha/property" +) + +func v2(cfg *declcfg.DeclarativeConfig) error { + uniqueBundles := map[string]*model.Bundle{} + m, err := declcfg.ConvertToModel(*cfg) + if err != nil { + return err + } + cfg.Packages = nil + cfg.Bundles = nil + + for _, pkg := range m { + head, err := pkg.DefaultChannel.Head() + if err != nil { + return err + } + + if head.PropertiesP == nil || len(head.PropertiesP.CSVMetadatas) == 0 { + return fmt.Errorf("no CSV metadata defined for package %s", pkg.Name) + } + csvMetadata := head.PropertiesP.CSVMetadatas[0] + + packageAnnotations := map[string]string{ + "operators.openshift.io/capabilities": csvMetadata.Annotations["capabilities"], + "operators.openshift.io/categories": csvMetadata.Annotations["categories"], + "operators.openshift.io/infrastructure-features": csvMetadata.Annotations["operators.openshift.io/infrastructure-features"], + "operators.openshift.io/valid-subscription": csvMetadata.Annotations["operators.openshift.io/valid-subscription"], + } + + v2p := declcfg.PackageV2{ + Schema: "olm.package.v2", + Package: pkg.Name, + DisplayName: csvMetadata.DisplayName, + ShortDescription: csvMetadata.Annotations["description"], + LongDescription: csvMetadata.Description, + Keywords: csvMetadata.Keywords, + Links: csvMetadata.Links, + Provider: csvMetadata.Provider, + Maintainers: csvMetadata.Maintainers, + Annotations: packageAnnotations, + } + cfg.PackageV2s = append(cfg.PackageV2s, v2p) + + if pkg.Icon != nil { + v2i := declcfg.PackageIcon{ + Schema: "olm.package.icon", + Package: pkg.Name, + Data: pkg.Icon.Data, + MediaType: pkg.Icon.MediaType, + } + cfg.PackageIcons = append(cfg.PackageIcons, v2i) + } + for _, ch := range pkg.Channels { + for _, b := range ch.Bundles { + uniqueBundles[b.Name] = b + } + } + } + + for _, b := range uniqueBundles { + v2b := declcfg.BundleV2{ + Schema: "olm.bundle.v2", + Package: b.Package.Name, + Name: b.Name, + + Version: b.Version.String(), + Release: 0, + Reference: fmt.Sprintf("docker://%s", b.Image), + RelatedReferences: make([]string, 0, len(b.RelatedImages)), + + Constraints: map[string][]json.RawMessage{}, + Properties: map[string][]json.RawMessage{}, + } + + for _, ri := range b.RelatedImages { + v2b.RelatedReferences = append(v2b.RelatedReferences, fmt.Sprintf("docker://%s", ri.Image)) + } + + for _, p := range b.Properties { + if p.Type == property.TypePackage || p.Type == property.TypeCSVMetadata || p.Type == property.TypeBundleObject { + continue + } + if isContraint(p) { + v2b.Constraints[p.Type] = append(v2b.Constraints[p.Type], p.Value) + } else { + v2b.Properties[p.Type] = append(v2b.Properties[p.Type], p.Value) + } + } + + if b.PropertiesP != nil && len(b.PropertiesP.CSVMetadatas) > 0 { + csvMetadata := b.PropertiesP.CSVMetadatas[0] + + desiredAnnotations := map[string]string{ + "createdAt": "operators.openshift.io/creationTimestamp", + "repository": "operators.openshift.io/repository", + "support": "operators.openshift.io/support", + "containerImage": "operators.openshift.io/image", + } + + bundleAnnotations := map[string]string{} + for fromKey, toKey := range desiredAnnotations { + if value, ok := csvMetadata.Annotations[fromKey]; ok { + bundleAnnotations[toKey] = value + } + } + v2b.Annotations = bundleAnnotations + } + cfg.BundleV2s = append(cfg.BundleV2s, v2b) + } + + for depIdx := range cfg.Deprecations { + for entryIdx := range cfg.Deprecations[depIdx].Entries { + e := &cfg.Deprecations[depIdx].Entries[entryIdx] + switch e.Reference.Schema { + case declcfg.SchemaPackage: + e.Reference.Schema = declcfg.SchemaPackageV2 + case declcfg.SchemaBundle: + e.Reference.Schema = declcfg.SchemaBundleV2 + } + } + } + return nil +} + +func isContraint(p property.Property) bool { + switch p.Type { + case property.TypeConstraint, property.TypeGVKRequired, property.TypePackageRequired: + return true + } + return false +} diff --git a/alpha/action/migrations/migrations.go b/alpha/action/migrations/migrations.go index 22ff86b74..32f02d09b 100644 --- a/alpha/action/migrations/migrations.go +++ b/alpha/action/migrations/migrations.go @@ -54,6 +54,7 @@ type Migrations struct { var allMigrations = []Migration{ newMigration(NoMigrations, "do nothing", func(_ *declcfg.DeclarativeConfig) error { return nil }), newMigration("bundle-object-to-csv-metadata", `migrates bundles' "olm.bundle.object" to "olm.csv.metadata"`, bundleObjectToCSVMetadata), + newMigration("v2", `migrate catalog to olm.package.v2, olm.bundle.v2, and olm.package.icon`, v2), } func NewMigrations(name string) (*Migrations, error) { diff --git a/alpha/declcfg/declcfg.go b/alpha/declcfg/declcfg.go index 7797baa49..b7a9b021e 100644 --- a/alpha/declcfg/declcfg.go +++ b/alpha/declcfg/declcfg.go @@ -6,13 +6,14 @@ import ( "errors" "fmt" - prettyunmarshaler "github.com/operator-framework/operator-registry/pkg/prettyunmarshaler" - "golang.org/x/text/cases" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" + "github.com/operator-framework/api/pkg/operators/v1alpha1" + "github.com/operator-framework/operator-registry/alpha/property" + prettyunmarshaler "github.com/operator-framework/operator-registry/pkg/prettyunmarshaler" ) const ( @@ -20,12 +21,20 @@ const ( SchemaChannel = "olm.channel" SchemaBundle = "olm.bundle" SchemaDeprecation = "olm.deprecations" + SchemaPackageV2 = "olm.package.v2" + SchemaBundleV2 = "olm.bundle.v2" + SchemaPackageIcon = "olm.package.icon" ) type DeclarativeConfig struct { Packages []Package - Channels []Channel - Bundles []Bundle + PackageV2s []PackageV2 + PackageIcons []PackageIcon + + Channels []Channel + Bundles []Bundle + BundleV2s []BundleV2 + Deprecations []Deprecation Others []Meta } @@ -92,6 +101,43 @@ type RelatedImage struct { Image string `json:"image"` } +type PackageV2 struct { + Schema string `json:"schema"` + Package string `json:"package"` + Annotations map[string]string `json:"annotations,omitempty"` + + DisplayName string `json:"displayName,omitempty"` + ShortDescription string `json:"shortDescription,omitempty"` + LongDescription string `json:"longDescription,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Links []v1alpha1.AppLink `json:"links,omitempty"` + Provider v1alpha1.AppLink `json:"provider,omitempty"` + Maintainers []v1alpha1.Maintainer `json:"maintainers,omitempty"` +} + +type PackageIcon struct { + Schema string `json:"schema"` + Package string `json:"package"` + Data []byte `json:"data"` + MediaType string `json:"mediaType"` +} + +type BundleV2 struct { + Schema string `json:"schema"` + Package string `json:"package"` + Name string `json:"name"` + Annotations map[string]string `json:"annotations,omitempty"` + + Version string `json:"version"` + Release uint32 `json:"release"` + + Reference string `json:"ref"` + RelatedReferences []string `json:"relatedReferences,omitempty"` + + Properties map[string][]json.RawMessage `json:"properties,omitempty"` + Constraints map[string][]json.RawMessage `json:"constraints,omitempty"` +} + type Deprecation struct { Schema string `json:"schema"` Package string `json:"package"` @@ -202,8 +248,11 @@ func extractUniqueMetaKeys(blobMap map[string]any, m *Meta) error { func (destination *DeclarativeConfig) Merge(src *DeclarativeConfig) { destination.Packages = append(destination.Packages, src.Packages...) + destination.PackageV2s = append(destination.PackageV2s, src.PackageV2s...) + destination.PackageIcons = append(destination.PackageIcons, src.PackageIcons...) destination.Channels = append(destination.Channels, src.Channels...) destination.Bundles = append(destination.Bundles, src.Bundles...) + destination.BundleV2s = append(destination.BundleV2s, src.BundleV2s...) destination.Others = append(destination.Others, src.Others...) destination.Deprecations = append(destination.Deprecations, src.Deprecations...) } diff --git a/alpha/declcfg/write.go b/alpha/declcfg/write.go index 9856c2e1e..39a45182f 100644 --- a/alpha/declcfg/write.go +++ b/alpha/declcfg/write.go @@ -2,11 +2,13 @@ package declcfg import ( "bytes" + "cmp" "encoding/json" "fmt" "io" "os" "path/filepath" + "slices" "sort" "strings" @@ -394,15 +396,26 @@ type encoder interface { Encode(interface{}) error } -func writeToEncoder(cfg DeclarativeConfig, enc encoder) error { - pkgNames := sets.NewString() - +func organizeByPackage(cfg DeclarativeConfig) map[string]DeclarativeConfig { + pkgNames := sets.New[string]() packagesByName := map[string][]Package{} for _, p := range cfg.Packages { pkgName := p.Name pkgNames.Insert(pkgName) packagesByName[pkgName] = append(packagesByName[pkgName], p) } + packageV2sByName := map[string][]PackageV2{} + for _, p := range cfg.PackageV2s { + pkgName := p.Package + pkgNames.Insert(pkgName) + packageV2sByName[pkgName] = append(packageV2sByName[pkgName], p) + } + packageIconsByPackage := map[string][]PackageIcon{} + for _, pi := range cfg.PackageIcons { + pkgName := pi.Package + pkgNames.Insert(pkgName) + packageIconsByPackage[pkgName] = append(packageIconsByPackage[pkgName], pi) + } channelsByPackage := map[string][]Channel{} for _, c := range cfg.Channels { pkgName := c.Package @@ -415,6 +428,12 @@ func writeToEncoder(cfg DeclarativeConfig, enc encoder) error { pkgNames.Insert(pkgName) bundlesByPackage[pkgName] = append(bundlesByPackage[pkgName], b) } + bundleV2sByPackage := map[string][]BundleV2{} + for _, b := range cfg.BundleV2s { + pkgName := b.Package + pkgNames.Insert(pkgName) + bundleV2sByPackage[pkgName] = append(bundleV2sByPackage[pkgName], b) + } othersByPackage := map[string][]Meta{} for _, o := range cfg.Others { pkgName := o.Package @@ -428,97 +447,73 @@ func writeToEncoder(cfg DeclarativeConfig, enc encoder) error { deprecationsByPackage[pkgName] = append(deprecationsByPackage[pkgName], d) } - for _, pName := range pkgNames.List() { - if len(pName) == 0 { - continue - } - pkgs := packagesByName[pName] - for _, p := range pkgs { - if err := enc.Encode(p); err != nil { - return err - } + fbcsByPackageName := make(map[string]DeclarativeConfig, len(pkgNames)) + for _, pkgName := range sets.List(pkgNames) { + fbcsByPackageName[pkgName] = DeclarativeConfig{ + Packages: packagesByName[pkgName], + PackageV2s: packageV2sByName[pkgName], + PackageIcons: packageIconsByPackage[pkgName], + Channels: channelsByPackage[pkgName], + Bundles: bundlesByPackage[pkgName], + BundleV2s: bundleV2sByPackage[pkgName], + Deprecations: deprecationsByPackage[pkgName], + Others: othersByPackage[pkgName], } + } + return fbcsByPackageName +} - channels := channelsByPackage[pName] - sort.Slice(channels, func(i, j int) bool { - return channels[i].Name < channels[j].Name - }) - for _, c := range channels { - if err := enc.Encode(c); err != nil { +func encodeAll[T any](values []T) func(encoder) error { + return func(enc encoder) error { + for _, v := range values { + if err := enc.Encode(v); err != nil { return err } } + return nil + } +} - bundles := bundlesByPackage[pName] - sort.Slice(bundles, func(i, j int) bool { - return bundles[i].Name < bundles[j].Name - }) - for _, b := range bundles { - if err := enc.Encode(b); err != nil { - return err +func writeToEncoder(cfg DeclarativeConfig, enc encoder) error { + byPackage := organizeByPackage(cfg) + + for _, pkgName := range sets.List(sets.KeySet(byPackage)) { + slices.SortFunc(byPackage[pkgName].Packages, func(i, j Package) int { return cmp.Compare(i.Name, j.Name) }) + slices.SortFunc(byPackage[pkgName].PackageV2s, func(i, j PackageV2) int { return cmp.Compare(i.Package, j.Package) }) + slices.SortFunc(byPackage[pkgName].PackageIcons, func(i, j PackageIcon) int { return cmp.Compare(i.Package, j.Package) }) + slices.SortFunc(byPackage[pkgName].Channels, func(i, j Channel) int { return cmp.Compare(i.Name, j.Name) }) + slices.SortFunc(byPackage[pkgName].Bundles, func(i, j Bundle) int { return cmp.Compare(i.Name, j.Name) }) + slices.SortFunc(byPackage[pkgName].BundleV2s, func(i, j BundleV2) int { return cmp.Compare(i.Name, j.Name) }) + slices.SortFunc(byPackage[pkgName].Deprecations, func(i, j Deprecation) int { return cmp.Compare(i.Package, j.Package) }) + slices.SortFunc(byPackage[pkgName].Others, func(i, j Meta) int { + if bySchema := cmp.Compare(i.Schema, j.Schema); bySchema != 0 { + return bySchema } - } - - others := othersByPackage[pName] - sort.SliceStable(others, func(i, j int) bool { - return others[i].Schema < others[j].Schema + return cmp.Compare(i.Name, j.Name) }) - for _, o := range others { - if err := enc.Encode(o); err != nil { - return err - } - } - - // - // Normally we would order the deprecations, but it really doesn't make sense since - // - there will be 0 or 1 of them for any given package - // - they have no other useful field for ordering - // - // validation is typically via conversion to a model.Model and invoking model.Package.Validate() - // It's possible that a user of the object could create a slice containing more then 1 - // Deprecation object for a package, and it would bypass validation if this - // function gets called without conversion. - // - deprecations := deprecationsByPackage[pName] - for _, d := range deprecations { - if err := enc.Encode(d); err != nil { + for _, f := range []func(enc encoder) error{ + encodeAll(byPackage[pkgName].Packages), + encodeAll(byPackage[pkgName].PackageV2s), + encodeAll(byPackage[pkgName].PackageIcons), + encodeAll(byPackage[pkgName].Channels), + encodeAll(byPackage[pkgName].Bundles), + encodeAll(byPackage[pkgName].BundleV2s), + encodeAll(byPackage[pkgName].Deprecations), + encodeAll(byPackage[pkgName].Others), + } { + if err := f(enc); err != nil { return err } } } - - for _, o := range othersByPackage[""] { - if err := enc.Encode(o); err != nil { - return err - } - } - return nil } type WriteFunc func(config DeclarativeConfig, w io.Writer) error func WriteFS(cfg DeclarativeConfig, rootDir string, writeFunc WriteFunc, fileExt string) error { - channelsByPackage := map[string][]Channel{} - for _, c := range cfg.Channels { - channelsByPackage[c.Package] = append(channelsByPackage[c.Package], c) - } - bundlesByPackage := map[string][]Bundle{} - for _, b := range cfg.Bundles { - bundlesByPackage[b.Package] = append(bundlesByPackage[b.Package], b) - } - - if err := os.MkdirAll(rootDir, 0777); err != nil { - return err - } - - for _, p := range cfg.Packages { - fcfg := DeclarativeConfig{ - Packages: []Package{p}, - Channels: channelsByPackage[p.Name], - Bundles: bundlesByPackage[p.Name], - } - pkgDir := filepath.Join(rootDir, p.Name) + for pkgName, fcfg := range organizeByPackage(cfg) { + pkgDir := filepath.Join(rootDir, pkgName) if err := os.MkdirAll(pkgDir, 0777); err != nil { return err }