introduce upgrade mechanism and add upgrade patch to v1 storage migration

pkg/apis/operator/v1alpha1/const.go

@@ -50,6 +50,7 @@ const (
 	LabelOperandName                = "operator.tekton.dev/operand-name"
 	DbSecretHash                    = "operator.tekton.dev/db-secret-hash"
 	DeploymentSpecHashValueLabelKey = "operator.tekton.dev/deployment-spec-applied-hash" // used to recreate pods, if there is a change detected in deployments spec
+	AppliedUpgradeVersionKey        = "operator.tekton.dev/applied-upgrade-version"      // used to monitor applied upgrade version, via upgrade package
 
 	UpgradePending = "upgrade pending"
 	Reinstalling   = "reinstalling"

pkg/apis/operator/v1alpha1/tektonconfig_lifecycle.go

@@ -107,3 +107,19 @@ func (tcs *TektonConfigStatus) GetVersion() string {
 func (tcs *TektonConfigStatus) SetVersion(version string) {
 	tcs.Version = version
 }
+
+// returns applied upgrade version
+func (tcs *TektonConfigStatus) GetAppliedUpgradeVersion() string {
+	if tcs.Annotations == nil {
+		return ""
+	}
+	return tcs.Annotations[AppliedUpgradeVersionKey]
+}
+
+// updates applied upgrade version
+func (tcs *TektonConfigStatus) SetAppliedUpgradeVersion(appliedUpgradeVersion string) {
+	if tcs.Annotations == nil {
+		tcs.Annotations = map[string]string{}
+	}
+	tcs.Annotations[AppliedUpgradeVersionKey] = appliedUpgradeVersion
+}

pkg/apis/operator/v1alpha1/tektonconfig_lifecycle_test.go

@@ -19,6 +19,7 @@ package v1alpha1
 import (
 	"testing"
 
+	"gotest.tools/v3/assert"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	apistest "knative.dev/pkg/apis/testing"
 )
@@ -96,3 +97,20 @@ func TestTektonConfigErrorPath(t *testing.T) {
 	}
 
 }
+
+func TestAppliedUpgradeVersion(t *testing.T) {
+	tc := &TektonConfig{}
+
+	// should return empty
+	assert.Equal(t, tc.Status.GetAppliedUpgradeVersion(), "")
+
+	// update applied upgrade version
+	tc.Status.SetAppliedUpgradeVersion("foo")
+	assert.Equal(t, tc.Status.GetAppliedUpgradeVersion(), "foo")
+	assert.Equal(t, tc.Status.Annotations[AppliedUpgradeVersionKey], "foo")
+
+	// update applied upgrade version
+	tc.Status.SetAppliedUpgradeVersion("bar")
+	assert.Equal(t, tc.Status.GetAppliedUpgradeVersion(), "bar")
+	assert.Equal(t, tc.Status.Annotations[AppliedUpgradeVersionKey], "bar")
+}

pkg/reconciler/shared/tektonconfig/controller.go

@@ -33,6 +33,7 @@ import (
 	tektonTriggerinformer "github.com/tektoncd/operator/pkg/client/injection/informers/operator/v1alpha1/tektontrigger"
 	tektonConfigreconciler "github.com/tektoncd/operator/pkg/client/injection/reconciler/operator/v1alpha1/tektonconfig"
 	"github.com/tektoncd/operator/pkg/reconciler/common"
+	"github.com/tektoncd/operator/pkg/reconciler/shared/tektonconfig/upgrade"
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
@@ -73,6 +74,8 @@ func NewExtensibleController(generator common.ExtensionGenerator) injection.Cont
 			manifest:          manifest,
 			operatorVersion:   operatorVer,
 		}
+		c.upgrade = upgrade.New(operatorVer, c.kubeClientSet, c.operatorClientSet, injection.GetConfig(ctx))
+
 		impl := tektonConfigreconciler.NewImpl(ctx, c)
 
 		logger.Info("Setting up event handlers for TektonConfig")

pkg/reconciler/shared/tektonconfig/tektonconfig.go

@@ -18,7 +18,6 @@ package tektonconfig
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 
 	mf "github.com/manifestival/manifestival"
@@ -29,9 +28,8 @@ import (
 	"github.com/tektoncd/operator/pkg/reconciler/shared/tektonconfig/chain"
 	"github.com/tektoncd/operator/pkg/reconciler/shared/tektonconfig/pipeline"
 	"github.com/tektoncd/operator/pkg/reconciler/shared/tektonconfig/trigger"
-	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"github.com/tektoncd/operator/pkg/reconciler/shared/tektonconfig/upgrade"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"knative.dev/pkg/logging"
 	pkgreconciler "knative.dev/pkg/reconciler"
@@ -47,6 +45,8 @@ type Reconciler struct {
 	extension       common.Extension
 	manifest        mf.Manifest
 	operatorVersion string
+	// performs pre and post upgrade operations
+	upgrade *upgrade.Upgrade
 }
 
 // Check that our Reconciler implements controller.Reconciler
@@ -193,7 +193,12 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, tc *v1alpha1.TektonConfi
 	tc.Status.MarkPostInstallComplete()
 
 	// Update the object for any spec changes
-	if _, err := r.operatorClientSet.OperatorV1alpha1().TektonConfigs().Update(ctx, tc, v1.UpdateOptions{}); err != nil {
+	if _, err := r.operatorClientSet.OperatorV1alpha1().TektonConfigs().Update(ctx, tc, metav1.UpdateOptions{}); err != nil {
+		return err
+	}
+
+	// run post upgrade
+	if err := r.upgrade.RunPostUpgrade(ctx); err != nil {
 		return err
 	}
 
@@ -212,37 +217,31 @@ func (r *Reconciler) markUpgrade(ctx context.Context, tc *v1alpha1.TektonConfig)
 		tc.Status.MarkPostInstallFailed(v1alpha1.UpgradePending)
 		tc.Status.MarkNotReady("Upgrade Pending")
 	}
-	if labels == nil {
-		labels = map[string]string{}
+	// update status
+	if _, err := r.operatorClientSet.OperatorV1alpha1().TektonConfigs().UpdateStatus(ctx, tc, metav1.UpdateOptions{}); err != nil {
+		return err
 	}
-	labels[v1alpha1.ReleaseVersionKey] = r.operatorVersion
-	tc.SetLabels(labels)
 
-	var chain v1alpha1.ChainProperties
-	cm, err := r.kubeClientSet.CoreV1().ConfigMaps(tc.Spec.GetTargetNamespace()).Get(ctx, "chains-config", metav1.GetOptions{})
-	if err != nil {
-		if apierrs.IsNotFound(err) {
-			chain = v1alpha1.ChainProperties{}
-		}
-	}
-	if len(cm.Data) > 0 {
-		jsonData, err := json.Marshal(cm.Data)
-		if err != nil {
-			return err
-		}
-		if err := json.Unmarshal(jsonData, &chain); err != nil {
-			return err
-		}
+	// run pre upgrade
+	if err := r.upgrade.RunPreUpgrade(ctx); err != nil {
+		return err
 	}
 
-	tc.Spec.Chain = v1alpha1.Chain{
-		ChainProperties: chain,
+	// update labels in tektonConfig CR
+	tcCR, err := r.operatorClientSet.OperatorV1alpha1().TektonConfigs().Get(ctx, v1alpha1.ConfigResourceName, metav1.GetOptions{})
+	if err != nil {
+		return err
 	}
 
-	// Update the object for any spec changes
-	if _, err := r.operatorClientSet.OperatorV1alpha1().TektonConfigs().Update(ctx,
-		tc, v1.UpdateOptions{}); err != nil {
+	labels = tcCR.GetLabels()
+	if labels == nil {
+		labels = map[string]string{}
+	}
+	labels[v1alpha1.ReleaseVersionKey] = r.operatorVersion
+	tcCR.SetLabels(labels)
+	if _, err := r.operatorClientSet.OperatorV1alpha1().TektonConfigs().Update(ctx, tcCR, metav1.UpdateOptions{}); err != nil {
 		return err
 	}
+
 	return v1alpha1.RECONCILE_AGAIN_ERR
 }

pkg/reconciler/shared/tektonconfig/upgrade/helper/storage_version_migrator.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2023 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"context"
+	"fmt"
+
+	"go.uber.org/zap"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"knative.dev/pkg/apiextensions/storageversion"
+)
+
+// performs crd storage version upgrade
+// lists all the resources and,
+// keeps only one storage version on the crd
+func MigrateStorageVersion(ctx context.Context, logger *zap.SugaredLogger, migrator *storageversion.Migrator, crdGroups []string) error {
+	logger.Infof("migrating %d group resources", len(crdGroups))
+
+	for _, crdGroupString := range crdGroups {
+		crdGroup := schema.ParseGroupResource(crdGroupString)
+		if crdGroup.Empty() {
+			logger.Errorf("unable to parse group version: %s", crdGroupString)
+			return fmt.Errorf("unable to parse group version: %s", crdGroupString)
+		}
+		logger.Info("migrating group resource ", crdGroup)
+		if err := migrator.Migrate(ctx, crdGroup); err != nil {
+			if apierrs.IsNotFound(err) {
+				logger.Infow("ignoring resource migration - unable to fetch a crd",
+					"crd", crdGroup, err,
+				)
+				continue
+			}
+			logger.Errorw("failed to migrate: ", err)
+			return err
+		}
+	}
+
+	return nil
+}

pkg/reconciler/shared/tektonconfig/upgrade/helper/storage_version_migrator_test.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2023 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	apix "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apixFake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	dynamicFake "k8s.io/client-go/dynamic/fake"
+	"knative.dev/pkg/apiextensions/storageversion"
+	"knative.dev/pkg/logging"
+)
+
+func TestMigrateStorageVersion(t *testing.T) {
+	fakeKind := schema.GroupKind{
+		Kind:  "Fake",
+		Group: "group.dev",
+	}
+
+	fakeGroup := schema.GroupResource{
+		Resource: "fakes",
+		Group:    "group.dev",
+	}
+
+	fakeCRD := &apix.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: fakeGroup.String(),
+		},
+		Spec: apix.CustomResourceDefinitionSpec{
+			Group: fakeKind.Group,
+			Versions: []apix.CustomResourceDefinitionVersion{
+				{Name: "v1alpha1", Served: true, Storage: false},
+				{Name: "v1beta1", Served: true, Storage: false},
+				{Name: "v1", Served: true, Storage: true},
+			},
+		},
+		Status: apix.CustomResourceDefinitionStatus{
+			StoredVersions: []string{
+				"v1alpha1",
+				"v1beta1",
+				"v1",
+			},
+		},
+	}
+
+	ctx := context.TODO()
+	resources := []runtime.Object{
+		fakeCrdResource("resource-1", "group.dev/v1alpha1"),
+		fakeCrdResource("resource-2", "group.dev/v1beta1"),
+		fakeCrdResource("resource-3", "group.dev/v1beta2"),
+		fakeCrdResource("resource-4", "group.dev/v1"),
+	}
+
+	dclient := dynamicFake.NewSimpleDynamicClient(runtime.NewScheme(), resources...)
+	cclient := apixFake.NewSimpleClientset(fakeCRD)
+	migrator := storageversion.NewMigrator(dclient, cclient)
+	logger := logging.FromContext(ctx)
+
+	// TEST
+	// expects only "v1"
+	err := MigrateStorageVersion(ctx, logger, migrator, []string{"fakes.group.dev", "unknown.group.dev"})
+	assert.NoError(t, err)
+	crd, err := cclient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, fakeCRD.GetName(), metav1.GetOptions{})
+	assert.NoError(t, err)
+	storageVersions := crd.Status.StoredVersions
+	assert.Len(t, storageVersions, 1)
+	assert.Equal(t, "v1", storageVersions[0])
+
+}
+
+func fakeCrdResource(name, apiVersion string) runtime.Object {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": apiVersion,
+			"kind":       "Fake",
+			"metadata": map[string]interface{}{
+				"name":      name,
+				"namespace": "default",
+			},
+		},
+	}
+}

pkg/reconciler/shared/tektonconfig/upgrade/post_upgrade.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2023 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"context"
+
+	"github.com/tektoncd/operator/pkg/client/clientset/versioned"
+	upgrade "github.com/tektoncd/operator/pkg/reconciler/shared/tektonconfig/upgrade/helper"
+	"go.uber.org/zap"
+	apixclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"knative.dev/pkg/apiextensions/storageversion"
+)
+
+// performs storage versions upgrade
+// lists all the resources and keeps only one storage version
+func upgradeStorageVersion(ctx context.Context, logger *zap.SugaredLogger, k8sClient kubernetes.Interface, operatorClient versioned.Interface, restConfig *rest.Config) error {
+	// resources to be upgraded
+	crdGroups := []string{
+		"clusterinterceptors.triggers.tekton.dev",
+		"clustertasks.tekton.dev",
+		"clustertriggerbindings.triggers.tekton.dev",
+		"customruns.tekton.dev",
+		"eventlisteners.triggers.tekton.dev",
+		"extensions.dashboard.tekton.dev",
+		"interceptors.triggers.tekton.dev",
+		"pipelineruns.tekton.dev",
+		"pipelines.tekton.dev",
+		"repositories.pipelinesascode.tekton.dev",
+		"resolutionrequests.resolution.tekton.dev",
+		"taskruns.tekton.dev",
+		"tasks.tekton.dev",
+		"triggerbindings.triggers.tekton.dev",
+		"triggers.triggers.tekton.dev",
+		"triggertemplates.triggers.tekton.dev",
+		"verificationpolicies.tekton.dev",
+	}
+
+	migrator := storageversion.NewMigrator(
+		dynamic.NewForConfigOrDie(restConfig),
+		apixclient.NewForConfigOrDie(restConfig),
+	)
+
+	return upgrade.MigrateStorageVersion(ctx, logger, migrator, crdGroups)
+}