From 791136ed546daf9c654383363bd835016562c9dc Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Mon, 14 Jun 2021 16:32:26 +0100 Subject: [PATCH 1/6] go fmt Signed-off-by: Jake Sanders --- pkg/cas/cas_test.go | 2 +- test/e2e/suite/leaderelection/leaderelection.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 test/e2e/suite/leaderelection/leaderelection.go diff --git a/pkg/cas/cas_test.go b/pkg/cas/cas_test.go index 76a0c3c7c..40deaad59 100644 --- a/pkg/cas/cas_test.go +++ b/pkg/cas/cas_test.go @@ -140,7 +140,7 @@ z5B9C4cjanJ67w== PemCertificate: `-----BEGIN CERTIFICATE----- leaf -----END CERTIFICATE-----`, -PemCertificateChain: []string{`-----BEGIN CERTIFICATE----- + PemCertificateChain: []string{`-----BEGIN CERTIFICATE----- intermediate2 -----END CERTIFICATE-----`, `-----BEGIN CERTIFICATE----- intermediate1 diff --git a/test/e2e/suite/leaderelection/leaderelection.go b/test/e2e/suite/leaderelection/leaderelection.go new file mode 100644 index 000000000..8709e504b --- /dev/null +++ b/test/e2e/suite/leaderelection/leaderelection.go @@ -0,0 +1 @@ +package leaderelection From 5cc732854eba8b0d8ac9a47ddc516e1e9b48e73d Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Mon, 14 Jun 2021 16:33:12 +0100 Subject: [PATCH 2/6] Use a stable ID for leader election Add a flag to allow users to customise the leader election ID Signed-off-by: Jake Sanders --- cmd/root.go | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 25de5c7ae..7d7aec87d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,10 +17,6 @@ limitations under the License. package cmd import ( - "crypto/rand" - "math/big" - "os" - cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -49,6 +45,7 @@ func init() { // Issuer flags rootCmd.PersistentFlags().String("metrics-addr", ":8080", "The address the metric endpoint binds to.") rootCmd.PersistentFlags().Bool("enable-leader-election", false, "Enable leader election for controller manager.") + rootCmd.PersistentFlags().String("leader-election-id", "cm-google-cas-issuer", "Enable leader election for controller manager.") rootCmd.PersistentFlags().String("cluster-resource-namespace", "cert-manager", "The namespace for secrets in which cluster-scoped resources are found.") rootCmd.PersistentFlags().Bool("disable-approval-check", false, "Don't check whether a CertificateRequest is approved before signing. For compatibility with cert-manager Date: Mon, 14 Jun 2021 16:33:28 +0100 Subject: [PATCH 3/6] e2e tests for leader election Signed-off-by: Jake Sanders --- test/e2e/suite/issuers/issuers.go | 7 +- .../suite/leaderelection/leaderelection.go | 95 +++++++++++++++++++ test/e2e/suite/suite.go | 1 + test/e2e/suite/validation/validation.go | 6 +- 4 files changed, 104 insertions(+), 5 deletions(-) diff --git a/test/e2e/suite/issuers/issuers.go b/test/e2e/suite/issuers/issuers.go index 6b77e60fc..fcd759b78 100644 --- a/test/e2e/suite/issuers/issuers.go +++ b/test/e2e/suite/issuers/issuers.go @@ -12,15 +12,16 @@ import ( "filippo.io/age" certmanagerv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1" cmmetav1 "github.com/jetstack/cert-manager/pkg/apis/meta/v1" - "github.com/jetstack/google-cas-issuer/test/e2e/framework" - "github.com/jetstack/google-cas-issuer/test/e2e/framework/config" - "github.com/jetstack/google-cas-issuer/test/e2e/util" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + + "github.com/jetstack/google-cas-issuer/test/e2e/framework" + "github.com/jetstack/google-cas-issuer/test/e2e/framework/config" + "github.com/jetstack/google-cas-issuer/test/e2e/util" ) const ( diff --git a/test/e2e/suite/leaderelection/leaderelection.go b/test/e2e/suite/leaderelection/leaderelection.go index 8709e504b..e7479147e 100644 --- a/test/e2e/suite/leaderelection/leaderelection.go +++ b/test/e2e/suite/leaderelection/leaderelection.go @@ -1 +1,96 @@ package leaderelection + +import ( + "context" + "fmt" + "k8s.io/apimachinery/pkg/util/json" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/jetstack/google-cas-issuer/test/e2e/framework" +) + +type leaderElectionAnnotation struct { + HolderIdentity string `json:"holderIdentity"` + LeaseDurationSeconds int `json:"leaseDurationSeconds"` + AcquireTime time.Time `json:"acquireTime"` + RenewTime time.Time `json:"renewTime"` + LeaderTransitions int `json:"leaderTransitions"` +} + +var _ = framework.CasesDescribe("leader election", func() { + f := framework.NewDefaultFramework("leader election") + It("Tests leader election", func() { + By("Waiting for all pods to be ready") + err := f.Helper().WaitForPodsReady(f.Config().Namespace, 10 * time.Second) + Expect(err).NotTo(HaveOccurred()) + By("Finding all cas issuer leader election config maps") + findAllCASIssuerConfigMaps := func() ([]*corev1.ConfigMap, error) { + configMapList, err := f.KubeClientSet.CoreV1().ConfigMaps("cert-manager").List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("couldn't retrieve a list of config maps: %w", err) + } + var configMaps []*corev1.ConfigMap + for _, cm := range configMapList.Items { + if leaderAnnotationJSON, found := cm.Annotations["control-plane.alpha.kubernetes.io/leader"]; found { + leaderInfo := new(leaderElectionAnnotation) + if err := json.Unmarshal([]byte(leaderAnnotationJSON), leaderInfo); err != nil { + return nil, err + } + if strings.HasPrefix(leaderInfo.HolderIdentity, "google-cas-issuer") { + configMaps = append(configMaps, &cm) + } + } + } + return configMaps, nil + } + + configMaps, err := findAllCASIssuerConfigMaps() + Expect(err).NotTo(HaveOccurred()) + By("Expecting only one config map pointing to a google CAS issuer") + Expect(configMaps).Should(HaveLen(1)) + + By("Retrieving the google cas issuer deployment") + deployment, err := f.KubeClientSet.AppsV1().Deployments("cert-manager").Get(context.TODO(), "google-cas-issuer", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Scaling google cas issuer to 3 replicas") + want3Replicas := int32(3) + newDeployment := deployment.DeepCopy() + newDeployment.Spec.Replicas = &want3Replicas + _, err = f.KubeClientSet.AppsV1().Deployments("cert-manager").Update(context.TODO(), newDeployment, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for all pods to be ready") + err = f.Helper().WaitForPodsReady(f.Config().Namespace, 10 * time.Second) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring there is still a single config map") + configMaps, err = findAllCASIssuerConfigMaps() + Expect(err).NotTo(HaveOccurred()) + Expect(configMaps).Should(HaveLen(1)) + + By("Scaling google cas issuer to 1 replicas") + deployment, err = f.KubeClientSet.AppsV1().Deployments("cert-manager").Get(context.TODO(), "google-cas-issuer", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + newDeployment = deployment.DeepCopy() + want1Replica := int32(1) + newDeployment.Spec.Replicas = &want1Replica + _, err = f.KubeClientSet.AppsV1().Deployments("cert-manager").Update(context.TODO(), newDeployment, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for all pods to be ready") + err = f.Helper().WaitForPodsReady(f.Config().Namespace, 10 * time.Second) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring there is still a single config map") + configMaps, err = findAllCASIssuerConfigMaps() + Expect(err).NotTo(HaveOccurred()) + Expect(configMaps).Should(HaveLen(1)) + }) +}) diff --git a/test/e2e/suite/suite.go b/test/e2e/suite/suite.go index 689095d6d..5ce3cabb0 100644 --- a/test/e2e/suite/suite.go +++ b/test/e2e/suite/suite.go @@ -2,5 +2,6 @@ package suite import ( _ "github.com/jetstack/google-cas-issuer/test/e2e/suite/issuers" + _ "github.com/jetstack/google-cas-issuer/test/e2e/suite/leaderelection" _ "github.com/jetstack/google-cas-issuer/test/e2e/suite/validation" ) diff --git a/test/e2e/suite/validation/validation.go b/test/e2e/suite/validation/validation.go index 5b32da51e..614f852d2 100644 --- a/test/e2e/suite/validation/validation.go +++ b/test/e2e/suite/validation/validation.go @@ -2,13 +2,15 @@ package validation import ( "context" - "github.com/jetstack/google-cas-issuer/test/e2e/framework" + "time" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" - "time" + + "github.com/jetstack/google-cas-issuer/test/e2e/framework" ) var _ = framework.CasesDescribe("validation", func() { From e632ea348029b3ad722a61d7b824046a77904fdc Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Mon, 14 Jun 2021 16:35:55 +0100 Subject: [PATCH 4/6] Update README in anticipation of v0.5.2 release Signed-off-by: Jake Sanders --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 883f9e382..afe1abfc9 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Visit the [GitHub releases](https://github.com/jetstack/google-cas-issuer/releas and copy the command, e.g. ```shell -kubectl apply -f https://github.com/jetstack/google-cas-issuer/releases/download/v0.5.0/google-cas-issuer-v0.5.0.yaml +kubectl apply -f https://github.com/jetstack/google-cas-issuer/releases/download/v0.5.2/google-cas-issuer-v0.5.2.yaml ``` You can then skip to the [Setting up Google Cloud IAM](#setting-up-google-cloud-iam) section. From 7f0e8c467f6b36ff8b1c8d614afd17cb1d40a28f Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Mon, 14 Jun 2021 16:39:59 +0100 Subject: [PATCH 5/6] Document leader election ID flag Signed-off-by: Jake Sanders --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 7d7aec87d..e1178abfe 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -45,7 +45,7 @@ func init() { // Issuer flags rootCmd.PersistentFlags().String("metrics-addr", ":8080", "The address the metric endpoint binds to.") rootCmd.PersistentFlags().Bool("enable-leader-election", false, "Enable leader election for controller manager.") - rootCmd.PersistentFlags().String("leader-election-id", "cm-google-cas-issuer", "Enable leader election for controller manager.") + rootCmd.PersistentFlags().String("leader-election-id", "cm-google-cas-issuer", "The ID of the leader election lock that the controller should attempt to acquire.") rootCmd.PersistentFlags().String("cluster-resource-namespace", "cert-manager", "The namespace for secrets in which cluster-scoped resources are found.") rootCmd.PersistentFlags().Bool("disable-approval-check", false, "Don't check whether a CertificateRequest is approved before signing. For compatibility with cert-manager Date: Mon, 14 Jun 2021 16:55:17 +0100 Subject: [PATCH 6/6] Use RetryOnConflict to avoid optimistic locking flakes in e2e Signed-off-by: Jake Sanders --- .../suite/leaderelection/leaderelection.go | 54 ++++++++++++------- test/e2e/suite/validation/validation.go | 7 --- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/test/e2e/suite/leaderelection/leaderelection.go b/test/e2e/suite/leaderelection/leaderelection.go index e7479147e..c856f718b 100644 --- a/test/e2e/suite/leaderelection/leaderelection.go +++ b/test/e2e/suite/leaderelection/leaderelection.go @@ -3,16 +3,16 @@ package leaderelection import ( "context" "fmt" - "k8s.io/apimachinery/pkg/util/json" "strings" "time" + "github.com/jetstack/google-cas-issuer/test/e2e/framework" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/jetstack/google-cas-issuer/test/e2e/framework" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/util/retry" ) type leaderElectionAnnotation struct { @@ -27,7 +27,7 @@ var _ = framework.CasesDescribe("leader election", func() { f := framework.NewDefaultFramework("leader election") It("Tests leader election", func() { By("Waiting for all pods to be ready") - err := f.Helper().WaitForPodsReady(f.Config().Namespace, 10 * time.Second) + err := f.Helper().WaitForPodsReady(f.Config().Namespace, 10*time.Second) Expect(err).NotTo(HaveOccurred()) By("Finding all cas issuer leader election config maps") findAllCASIssuerConfigMaps := func() ([]*corev1.ConfigMap, error) { @@ -55,19 +55,25 @@ var _ = framework.CasesDescribe("leader election", func() { By("Expecting only one config map pointing to a google CAS issuer") Expect(configMaps).Should(HaveLen(1)) - By("Retrieving the google cas issuer deployment") - deployment, err := f.KubeClientSet.AppsV1().Deployments("cert-manager").Get(context.TODO(), "google-cas-issuer", metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - By("Scaling google cas issuer to 3 replicas") - want3Replicas := int32(3) - newDeployment := deployment.DeepCopy() - newDeployment.Spec.Replicas = &want3Replicas - _, err = f.KubeClientSet.AppsV1().Deployments("cert-manager").Update(context.TODO(), newDeployment, metav1.UpdateOptions{}) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + deployment, err := f.KubeClientSet.AppsV1().Deployments("cert-manager").Get(context.TODO(), "google-cas-issuer", metav1.GetOptions{}) + if err != nil { + return err + } + want3Replicas := int32(3) + newDeployment := deployment.DeepCopy() + newDeployment.Spec.Replicas = &want3Replicas + _, err = f.KubeClientSet.AppsV1().Deployments("cert-manager").Update(context.TODO(), newDeployment, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }) Expect(err).NotTo(HaveOccurred()) By("Waiting for all pods to be ready") - err = f.Helper().WaitForPodsReady(f.Config().Namespace, 10 * time.Second) + err = f.Helper().WaitForPodsReady(f.Config().Namespace, 10*time.Second) Expect(err).NotTo(HaveOccurred()) By("Ensuring there is still a single config map") @@ -76,16 +82,24 @@ var _ = framework.CasesDescribe("leader election", func() { Expect(configMaps).Should(HaveLen(1)) By("Scaling google cas issuer to 1 replicas") - deployment, err = f.KubeClientSet.AppsV1().Deployments("cert-manager").Get(context.TODO(), "google-cas-issuer", metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - newDeployment = deployment.DeepCopy() - want1Replica := int32(1) - newDeployment.Spec.Replicas = &want1Replica - _, err = f.KubeClientSet.AppsV1().Deployments("cert-manager").Update(context.TODO(), newDeployment, metav1.UpdateOptions{}) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + deployment, err := f.KubeClientSet.AppsV1().Deployments("cert-manager").Get(context.TODO(), "google-cas-issuer", metav1.GetOptions{}) + if err != nil { + return err + } + newDeployment := deployment.DeepCopy() + want1Replica := int32(1) + newDeployment.Spec.Replicas = &want1Replica + _, err = f.KubeClientSet.AppsV1().Deployments("cert-manager").Update(context.TODO(), newDeployment, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }) Expect(err).NotTo(HaveOccurred()) By("Waiting for all pods to be ready") - err = f.Helper().WaitForPodsReady(f.Config().Namespace, 10 * time.Second) + err = f.Helper().WaitForPodsReady(f.Config().Namespace, 10*time.Second) Expect(err).NotTo(HaveOccurred()) By("Ensuring there is still a single config map") diff --git a/test/e2e/suite/validation/validation.go b/test/e2e/suite/validation/validation.go index 614f852d2..993c9518a 100644 --- a/test/e2e/suite/validation/validation.go +++ b/test/e2e/suite/validation/validation.go @@ -2,7 +2,6 @@ package validation import ( "context" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -27,12 +26,6 @@ var _ = framework.CasesDescribe("validation", func() { Expect(err).NotTo(HaveOccurred()) }) - It("All pods in ns cert-manager are ready", func() { - By("waiting until all pods have ready condition") - err := f.Helper().WaitForPodsReady("cert-manager", 2*time.Minute) - Expect(err).NotTo(HaveOccurred()) - }) - It("Has the google-cas-issuer CRDs installed", func() { By("using the dynamic client to create a google-cas-issuer") casYAML := `apiVersion: cas-issuer.jetstack.io/v1beta1