From 1cce039d71dd80a399b93056c46cd13001eefbef Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Mon, 7 Oct 2024 23:46:30 +0200 Subject: [PATCH 01/15] versions: add k8s 1.31, remove k8s 1.28 --- .github/workflows/e2e-test-daily.yml | 2 +- .github/workflows/e2e-test-internal-lb.yml | 2 +- .../workflows/e2e-test-marketplace-image.yml | 2 +- .github/workflows/e2e-test-release.yml | 20 +-- .../workflows/e2e-test-terraform-provider.yml | 2 +- .github/workflows/e2e-test-weekly.yml | 30 ++--- .github/workflows/e2e-test.yml | 2 +- docs/docs/architecture/versions.md | 2 +- docs/docs/reference/cli.md | 2 +- internal/constellation/kubecmd/BUILD.bazel | 5 +- internal/constellation/kubecmd/kubecmd.go | 56 ++++---- .../constellation/kubecmd/kubecmd_test.go | 124 ++++++++++++++++++ internal/versions/versions.go | 52 ++++---- .../docs/resources/cluster.md | 2 +- 14 files changed, 212 insertions(+), 91 deletions(-) diff --git a/.github/workflows/e2e-test-daily.yml b/.github/workflows/e2e-test-daily.yml index a6ac4fa3f7..16573d50e8 100644 --- a/.github/workflows/e2e-test-daily.yml +++ b/.github/workflows/e2e-test-daily.yml @@ -45,7 +45,7 @@ jobs: fail-fast: false max-parallel: 5 matrix: - kubernetesVersion: ["1.28"] # should be default + kubernetesVersion: ["1.29"] # should be default attestationVariant: ["gcp-sev-es", "gcp-sev-snp", "azure-sev-snp", "azure-tdx", "aws-sev-snp"] refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"] test: ["sonobuoy quick"] diff --git a/.github/workflows/e2e-test-internal-lb.yml b/.github/workflows/e2e-test-internal-lb.yml index b9a27949c3..a39b918c9f 100644 --- a/.github/workflows/e2e-test-internal-lb.yml +++ b/.github/workflows/e2e-test-internal-lb.yml @@ -41,7 +41,7 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.28" + default: "1.29" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-marketplace-image.yml b/.github/workflows/e2e-test-marketplace-image.yml index 3338c13847..9bf72a5474 100644 --- a/.github/workflows/e2e-test-marketplace-image.yml +++ b/.github/workflows/e2e-test-marketplace-image.yml @@ -41,7 +41,7 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.28" + default: "1.29" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-release.yml b/.github/workflows/e2e-test-release.yml index 598c4091ae..251fac11c6 100644 --- a/.github/workflows/e2e-test-release.yml +++ b/.github/workflows/e2e-test-release.yml @@ -73,53 +73,53 @@ jobs: - test: "sonobuoy full" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-tdx" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-tdx" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" diff --git a/.github/workflows/e2e-test-terraform-provider.yml b/.github/workflows/e2e-test-terraform-provider.yml index 585ffe6b90..5d8d6a32c5 100644 --- a/.github/workflows/e2e-test-terraform-provider.yml +++ b/.github/workflows/e2e-test-terraform-provider.yml @@ -41,7 +41,7 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.28" + default: "1.29" required: true releaseVersion: description: "Version of a released provider to download. Leave empty to build the provider from the checked out ref." diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml index 29f0f32be7..f548bcccb0 100644 --- a/.github/workflows/e2e-test-weekly.yml +++ b/.github/workflows/e2e-test-weekly.yml @@ -89,53 +89,53 @@ jobs: - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-tdx" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-tdx" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" @@ -290,27 +290,27 @@ jobs: - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "azure-tdx" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" runs-on: ubuntu-24.04 diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c06c8eefff..1e413f48d6 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -43,7 +43,7 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.29" + default: "1.30" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/docs/docs/architecture/versions.md b/docs/docs/architecture/versions.md index 30d9d28e24..59f9785fff 100644 --- a/docs/docs/architecture/versions.md +++ b/docs/docs/architecture/versions.md @@ -16,6 +16,6 @@ Subsequent Constellation releases drop support for the oldest (and deprecated) K The following Kubernetes versions are currently supported: -* v1.28.13 * v1.29.8 * v1.30.4 +* v1.31.1 diff --git a/docs/docs/reference/cli.md b/docs/docs/reference/cli.md index 99acef5203..ebddf84c9a 100644 --- a/docs/docs/reference/cli.md +++ b/docs/docs/reference/cli.md @@ -80,7 +80,7 @@ constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] ``` -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used -h, --help help for generate - -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.29") + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.30") -t, --tags strings additional tags for created resources given a list of key=value ``` diff --git a/internal/constellation/kubecmd/BUILD.bazel b/internal/constellation/kubecmd/BUILD.bazel index 71bae3c32d..aca26d0bb2 100644 --- a/internal/constellation/kubecmd/BUILD.bazel +++ b/internal/constellation/kubecmd/BUILD.bazel @@ -30,8 +30,11 @@ go_library( "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/runtime", "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_apimachinery//pkg/runtime/serializer/json", "@io_k8s_client_go//util/retry", - "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/scheme", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta4", "@io_k8s_sigs_yaml//:yaml", ], ) diff --git a/internal/constellation/kubecmd/kubecmd.go b/internal/constellation/kubecmd/kubecmd.go index 1ebf992654..a193f2528e 100644 --- a/internal/constellation/kubecmd/kubecmd.go +++ b/internal/constellation/kubecmd/kubecmd.go @@ -42,9 +42,11 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/client-go/util/retry" - kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" - "sigs.k8s.io/yaml" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" + kubeadmv1beta4 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4" ) // ErrInProgress signals that an upgrade is in progress inside the cluster. @@ -234,9 +236,23 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At // ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs. // Empty strings are ignored, existing SANs are preserved. func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error { - clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx) - if err != nil { - return fmt.Errorf("getting ClusterConfig: %w", err) + var kubeadmConfig *corev1.ConfigMap + if err := k.retryAction(ctx, func(ctx context.Context) error { + var err error + kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) + return err + }); err != nil { + return fmt.Errorf("retrieving current kubeadm-config: %w", err) + } + + clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] + if !ok { + return errors.New("ClusterConfiguration missing from kubeadm-config") + } + + var clusterConfiguration kubeadm.ClusterConfiguration + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { + return fmt.Errorf("decoding cluster configuration data: %w", err) } existingSANs := make(map[string]struct{}) @@ -264,7 +280,10 @@ func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNa clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) sort.Strings(clusterConfiguration.APIServer.CertSANs) - newConfigYAML, err := yaml.Marshal(clusterConfiguration) + opt := k8sjson.SerializerOptions{Yaml: true} + serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) + encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) + newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) if err != nil { return fmt.Errorf("marshaling ClusterConfiguration: %w", err) } @@ -316,31 +335,6 @@ func (k *KubeCmd) getConstellationVersion(ctx context.Context) (updatev1alpha1.N return nodeVersion, nil } -// getClusterConfiguration fetches the kubeadm-config configmap from the cluster, extracts the config -// and returns both the full configmap and the ClusterConfiguration. -func (k *KubeCmd) getClusterConfiguration(ctx context.Context) (kubeadmv1beta3.ClusterConfiguration, *corev1.ConfigMap, error) { - var existingConf *corev1.ConfigMap - if err := k.retryAction(ctx, func(ctx context.Context) error { - var err error - existingConf, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) - return err - }); err != nil { - return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("retrieving current kubeadm-config: %w", err) - } - - clusterConf, ok := existingConf.Data[constants.ClusterConfigurationKey] - if !ok { - return kubeadmv1beta3.ClusterConfiguration{}, nil, errors.New("ClusterConfiguration missing from kubeadm-config") - } - - var existingClusterConfig kubeadmv1beta3.ClusterConfiguration - if err := yaml.Unmarshal([]byte(clusterConf), &existingClusterConfig); err != nil { - return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("unmarshaling ClusterConfiguration: %w", err) - } - - return existingClusterConfig, existingConf, nil -} - // applyComponentsCM applies the k8s components ConfigMap to the cluster. func (k *KubeCmd) applyComponentsCM(ctx context.Context, components *corev1.ConfigMap) error { if err := k.retryAction(ctx, func(ctx context.Context) error { diff --git a/internal/constellation/kubecmd/kubecmd_test.go b/internal/constellation/kubecmd/kubecmd_test.go index 74e9562c15..79eef03c34 100644 --- a/internal/constellation/kubecmd/kubecmd_test.go +++ b/internal/constellation/kubecmd/kubecmd_test.go @@ -676,6 +676,50 @@ func TestRetryAction(t *testing.T) { } } +func TestExtendClusterConfigCertSANs(t *testing.T) { + ctx := context.Background() + + testCases := map[string]struct { + clusterConfig string + }{ + "kubeadmv1beta3.ClusterConfiguration": { + clusterConfig: kubeadmClusterConfigurationV1Beta3, + }, + "kubeadmv1beta4.ClusterConfiguration": { + clusterConfig: kubeadmClusterConfigurationV1Beta4, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + kubectl := &fakeConfigMapClient{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": tc.clusterConfig}}, + }, + } + cmd := &KubeCmd{ + kubectl: kubectl, + log: logger.NewTest(t), + retryInterval: time.Millisecond, + } + + err := cmd.ExtendClusterConfigCertSANs(ctx, []string{"example.com"}) + require.NoError(err) + + cm := kubectl.configMaps["kubeadm-config"] + require.NotNil(cm) + cc := cm.Data["ClusterConfiguration"] + require.NotNil(cc) + // Verify that SAN was added. + assert.Contains(cc, "example.com") + // Verify that config was written in v1beta4, regardless of the version read. + assert.Contains(cc, "kubeadm.k8s.io/v1beta4") + }) + } +} + type fakeUnstructuredClient struct { mock.Mock } @@ -835,3 +879,83 @@ func supportedValidK8sVersions() (res []versions.ValidK8sVersion) { } return } + +var kubeadmClusterConfigurationV1Beta3 = ` +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +apiServer: + certSANs: + - 127.0.0.1 + extraArgs: + kubelet-certificate-authority: /etc/kubernetes/pki/ca.crt + profiling: "false" + extraVolumes: + - hostPath: /var/log/kubernetes/audit/ + mountPath: /var/log/kubernetes/audit/ + name: audit-log + pathType: DirectoryOrCreate +certificatesDir: /etc/kubernetes/pki +clusterName: test-55bbf58d +controlPlaneEndpoint: 34.149.125.227:6443 +controllerManager: + extraArgs: + cloud-provider: external +dns: + disabled: true +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kubernetesVersion: v1.31.1 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: + disabled: true +scheduler: + extraArgs: + profiling: "false" +` + +var kubeadmClusterConfigurationV1Beta4 = ` +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ClusterConfiguration +apiServer: + certSANs: + - 127.0.0.1 + extraArgs: + - name: kubelet-certificate-authority + value: /etc/kubernetes/pki/ca.crt + - name: profiling + value: "false" + extraVolumes: + - hostPath: /var/log/kubernetes/audit/ + mountPath: /var/log/kubernetes/audit/ + name: audit-log + pathType: DirectoryOrCreate +certificatesDir: /etc/kubernetes/pki +clusterName: test-55bbf58d +controlPlaneEndpoint: 34.149.125.227:6443 +controllerManager: + extraArgs: + - name: cloud-provider + value: external +dns: + disabled: true +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kubernetesVersion: v1.31.1 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: + disabled: true +scheduler: + extraArgs: + - name: profiling + value: "false" +` diff --git a/internal/versions/versions.go b/internal/versions/versions.go index 0621b5d744..2ff9889f87 100644 --- a/internal/versions/versions.go +++ b/internal/versions/versions.go @@ -100,12 +100,12 @@ func ResolveK8sPatchVersion(k8sVersion string) (string, error) { // supported patch version as PATCH. func k8sVersionFromMajorMinor(version string) string { switch version { - case semver.MajorMinor(string(V1_28)): - return string(V1_28) case semver.MajorMinor(string(V1_29)): return string(V1_29) case semver.MajorMinor(string(V1_30)): return string(V1_30) + case semver.MajorMinor(string(V1_31)): + return string(V1_31) default: return "" } @@ -181,14 +181,14 @@ const ( // currently supported versions. //nolint:revive - V1_28 ValidK8sVersion = "v1.28.13" // renovate:kubernetes-release - //nolint:revive V1_29 ValidK8sVersion = "v1.29.8" // renovate:kubernetes-release //nolint:revive V1_30 ValidK8sVersion = "v1.30.4" // renovate:kubernetes-release + //nolint:revive + V1_31 ValidK8sVersion = "v1.31.1" // renovate:kubernetes-release // Default k8s version deployed by Constellation. - Default ValidK8sVersion = V1_29 + Default ValidK8sVersion = V1_30 ) // Regenerate the hashes by running go generate. @@ -197,49 +197,49 @@ const ( // VersionConfigs holds download URLs for all required kubernetes components for every supported version. var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ - V1_28: { - ClusterVersion: "v1.28.13", // renovate:kubernetes-release + V1_31: { + ClusterVersion: "v1.31.1", // renovate:kubernetes-release KubernetesComponents: components.Components{ { - Url: "https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz", // renovate:cni-plugins-release - Hash: "sha256:c2485ddb3ffc176578ae30ae58137f0b88e50f7c7f2af7d53a569276b2949a33", + Url: "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz", // renovate:cni-plugins-release + Hash: "sha256:77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b", InstallPath: constants.CniPluginsDir, Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.28.0/crictl-v1.28.0-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:8dc78774f7cbeaf787994d386eec663f0a3cf24de1ea4893598096cb39ef2508", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:0a03ba6b1e4c253d63627f8d210b2ea07675a8712587e697657b236d06d7d231", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.28.13/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:9b9cc3a19551ade6f3d98ad3acf0a2b65a27ef575bd089f115f8bb80791f3900", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:50619fff95bdd7e690c049cc083f495ae0e7c66d0cdf6a8bcad298af5fe28438", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.28.13/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:f23e9586811312998bc5e8847f6df52fc04809aed8c2c2fd750f2c42b3f87192", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:b3f92d19d482359116dd9ee9c0a10cb86e32a2a2aef79b853d5f07d6a093b0df", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.28.13/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:d7d363dd5a4c95444329bc5239b8718ebe84a043052958b2f15ee2feef9a28c6", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:57b514a7facce4ee62c93b8dc21fda8cf62ef3fed22e44ffc9d167eab843b2ae", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI4LjEzQHNoYTI1Njo3ZDJjOTI1NmFkNTc2YTBiMzc0NWI3NDllZmU3ZjRmYThiMjc2ZWM3ZWY0NDhmYzBmNDU3OTRjYTc4ZWI4NjI1In1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMxLjFAc2hhMjU2OjI0MDljMjNkYmI1YTJiN2E4MWFkYmIxODRkM2VhYzQzYWM2NTNlOWI5N2E3YzBlZTEyMWI4OWJiM2VmNjFmZGIifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI4LjEzQHNoYTI1NjplN2I0NGMxNzQxZmUxODAyZDE1OWZmZGJkMGQxZjc4ZDQ4YTQxODVkN2ZiMWNkZjhhMTEyZmJiNTA2OTZmN2UxIn1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMxLjFAc2hhMjU2OjlmOWRhNWIyN2UwM2Y4OTU5OWNjNDBiYTg5MTUwYWViZjNiNGNmZjAwMWU2ZGI2ZDk5ODY3NGIzNDE4MWUxYTEifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI4LjEzQHNoYTI1NjplZmViNzkxNzE4ZjRiOWM2MmJkNjgzZjViNDAzZGE1MjBmMzY1MWNiMzZhZDlmODAwZTBmOThiNTk1YmVhZmE0In1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMxLjFAc2hhMjU2Ojk2OWE3ZTk2MzQwZjNhOTI3YjNkNjUyNTgyZWRlYzJkNmQ4MmEwODM4NzFkODFlZjUwNjRiN2VkYWFiNDMwZDAifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -249,20 +249,20 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.9@sha256:168905b591796fbd07cb35cd0e3f206fdb7efb30e325c9bf7fa70d1b48989f73", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.31.0@sha256:c4658f5c583e85ef58a096ac9901a8ced3cb9699d8eeceb59d3bb54c5d0db72f", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.28.12@sha256:5ba85a312fdb6b65d4267a1e42090aaa9ebb69f44858d9f9f806a317a9260530", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.31.1@sha256:b5aa55a7e9d38137f7fcd0adc9335b06e7c96061764addd7e6bb9f86403f0110", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.28.12@sha256:e4a4ddb678f625399b85f4becff16ef07a06da7f5f4673ca21cfb356c370930d", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.1@sha256:e9b522399e4ec6bc4ce90c173e59db135d742de7b16f0f5454b4d88ba78a98c7", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. - CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v28.10.0@sha256:f3b6fa7faea27b4a303c91b3bc7ee192b050e21e27579e9f3da90ae4ba38e626", // renovate:container + CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.0@sha256:64d2d5d4d2b5fb426c307c64ada9a61b64e797b56d9768363f145f2bd957998f", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. - CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container + CloudControllerManagerImageOpenStack: "registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.31.1@sha256:72cc0d22b83c613df809d8134e50404171513d92287e63e2313d9ad7e1ed630e", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.28.6@sha256:acfc7fe7543f4cf2fcf8156145925bee76eb6c602bb0b8e155456c6818fe8335", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.31.0@sha256:6d4c51c35f344d230341d71bb6d35f2c2f0c0a6f205a7887ae44e6d852fb5b5f", // renovate:container }, V1_29: { ClusterVersion: "v1.29.8", // renovate:kubernetes-release diff --git a/terraform-provider-constellation/docs/resources/cluster.md b/terraform-provider-constellation/docs/resources/cluster.md index 542200750b..5befe11470 100644 --- a/terraform-provider-constellation/docs/resources/cluster.md +++ b/terraform-provider-constellation/docs/resources/cluster.md @@ -69,7 +69,7 @@ resource "constellation_cluster" "azure_example" { See the [full list of CSPs](https://docs.edgeless.systems/constellation/overview/clouds) that Constellation supports. - `image` (Attributes) Constellation OS Image to use on the nodes. (see [below for nested schema](#nestedatt--image)) - `init_secret` (String) Secret used for initialization of the cluster. -- `kubernetes_version` (String) The Kubernetes version to use for the cluster. The supported versions are [v1.28.13 v1.29.8 v1.30.4]. +- `kubernetes_version` (String) The Kubernetes version to use for the cluster. The supported versions are [v1.29.8 v1.30.4 v1.31.1]. - `master_secret` (String) Hex-encoded 32-byte master secret for the cluster. - `master_secret_salt` (String) Hex-encoded 32-byte master secret salt for the cluster. - `measurement_salt` (String) Hex-encoded 32-byte measurement salt for the cluster. From 86fcd1cc2b1f6adb9ebe692732ad683879b847d7 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 9 Oct 2024 09:22:17 +0200 Subject: [PATCH 02/15] e2e: set default k8s version for daily to 1.30 --- .github/workflows/e2e-test-daily.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test-daily.yml b/.github/workflows/e2e-test-daily.yml index 16573d50e8..5cd837a050 100644 --- a/.github/workflows/e2e-test-daily.yml +++ b/.github/workflows/e2e-test-daily.yml @@ -45,7 +45,7 @@ jobs: fail-fast: false max-parallel: 5 matrix: - kubernetesVersion: ["1.29"] # should be default + kubernetesVersion: ["1.30"] # This should correspond to the current default k8s minor. attestationVariant: ["gcp-sev-es", "gcp-sev-snp", "azure-sev-snp", "azure-tdx", "aws-sev-snp"] refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"] test: ["sonobuoy quick"] From c6ff4ecb73cc41f78514275c686fbb10350799ed Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 9 Oct 2024 09:22:42 +0200 Subject: [PATCH 03/15] e2e: remove defaults for required arguments --- .github/workflows/e2e-test-internal-lb.yml | 1 - .github/workflows/e2e-test-marketplace-image.yml | 1 - .github/workflows/e2e-test-terraform-provider.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/e2e-test-internal-lb.yml b/.github/workflows/e2e-test-internal-lb.yml index a39b918c9f..90940f4e00 100644 --- a/.github/workflows/e2e-test-internal-lb.yml +++ b/.github/workflows/e2e-test-internal-lb.yml @@ -41,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.29" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-marketplace-image.yml b/.github/workflows/e2e-test-marketplace-image.yml index 9bf72a5474..81f8b0a69d 100644 --- a/.github/workflows/e2e-test-marketplace-image.yml +++ b/.github/workflows/e2e-test-marketplace-image.yml @@ -41,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.29" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-terraform-provider.yml b/.github/workflows/e2e-test-terraform-provider.yml index 5d8d6a32c5..1ea5de7728 100644 --- a/.github/workflows/e2e-test-terraform-provider.yml +++ b/.github/workflows/e2e-test-terraform-provider.yml @@ -41,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.29" required: true releaseVersion: description: "Version of a released provider to download. Leave empty to build the provider from the checked out ref." From 2980a99a2d302252385716526a35fd1393d00a03 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 9 Oct 2024 09:14:13 +0200 Subject: [PATCH 04/15] versions: move 1.31 to the end of the list This reverts commit e957e90f85ffd87165f73650155fb323efe04822. --- internal/versions/versions.go | 118 +++++++++++++++++----------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/internal/versions/versions.go b/internal/versions/versions.go index 2ff9889f87..fe600bca23 100644 --- a/internal/versions/versions.go +++ b/internal/versions/versions.go @@ -197,49 +197,49 @@ const ( // VersionConfigs holds download URLs for all required kubernetes components for every supported version. var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ - V1_31: { - ClusterVersion: "v1.31.1", // renovate:kubernetes-release + V1_29: { + ClusterVersion: "v1.29.8", // renovate:kubernetes-release KubernetesComponents: components.Components{ { - Url: "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz", // renovate:cni-plugins-release - Hash: "sha256:77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b", + Url: "https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz", // renovate:cni-plugins-release + Hash: "sha256:c2485ddb3ffc176578ae30ae58137f0b88e50f7c7f2af7d53a569276b2949a33", InstallPath: constants.CniPluginsDir, Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:0a03ba6b1e4c253d63627f8d210b2ea07675a8712587e697657b236d06d7d231", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:50619fff95bdd7e690c049cc083f495ae0e7c66d0cdf6a8bcad298af5fe28438", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:df6e130928403af8b4f49f1197e26f2873a147cd0e23aa6597a26c982c652ae0", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:b3f92d19d482359116dd9ee9c0a10cb86e32a2a2aef79b853d5f07d6a093b0df", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:fe054355e0ae8dc35d868a3d3bc408ccdff0969c20bf7a231ae9b71484e41be3", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:57b514a7facce4ee62c93b8dc21fda8cf62ef3fed22e44ffc9d167eab843b2ae", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:038454e0d79748aab41668f44ca6e4ac8affd1895a94f592b9739a0ae2a5f06a", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMxLjFAc2hhMjU2OjI0MDljMjNkYmI1YTJiN2E4MWFkYmIxODRkM2VhYzQzYWM2NTNlOWI5N2E3YzBlZTEyMWI4OWJiM2VmNjFmZGIifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI5LjhAc2hhMjU2OjZmNzJmYTkyNmM5YjA1ZTEwNjI5ZmUxYTA5MmZkMjhkY2Q2NWI0ZmRmZDBjYzdiZDU1Zjg1YTU3YTZiYTFmYTUifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMxLjFAc2hhMjU2OjlmOWRhNWIyN2UwM2Y4OTU5OWNjNDBiYTg5MTUwYWViZjNiNGNmZjAwMWU2ZGI2ZDk5ODY3NGIzNDE4MWUxYTEifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI5LjhAc2hhMjU2OjZmMjdkNjNkZWQyMDYxNGM2ODU1NGI0NzdjZDdhNzhlZGE3OGE0OThhOTJiZmU4OTM1Y2Y5NjRjYTViNzRkMGIifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMxLjFAc2hhMjU2Ojk2OWE3ZTk2MzQwZjNhOTI3YjNkNjUyNTgyZWRlYzJkNmQ4MmEwODM4NzFkODFlZjUwNjRiN2VkYWFiNDMwZDAifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI5LjhAc2hhMjU2OmRhNzRhNjY2NzVkOTVlMzllYzI1ZGE1ZTcwNzI5ZGE3NDZkMGZhMGIxNWVlMGRhODcyYWM5ODA1MTliYzI4YmQifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -249,23 +249,23 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.31.0@sha256:c4658f5c583e85ef58a096ac9901a8ced3cb9699d8eeceb59d3bb54c5d0db72f", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.29.6@sha256:8074b8828a33fb273833e8fd374dda6a0ab10335ae8e19684fbd61eeff7d3594", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.31.1@sha256:b5aa55a7e9d38137f7fcd0adc9335b06e7c96061764addd7e6bb9f86403f0110", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.29.10@sha256:2e4d2c0bb15f2987cb7abb52e86c9b62504226fdbf19d5b597ac4707adb52a56", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.1@sha256:e9b522399e4ec6bc4ce90c173e59db135d742de7b16f0f5454b4d88ba78a98c7", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.29.10@sha256:ef27f4e2c104614fa44265d6e0a193bf1f9b5abda542f5b64efb868d3796ffc8", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. - CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.0@sha256:64d2d5d4d2b5fb426c307c64ada9a61b64e797b56d9768363f145f2bd957998f", // renovate:container + CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v29.5.1@sha256:ebbc6f5755725b6c2c81ca1d1580e2feba83572c41608b739c50f85b2e5de936", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. - CloudControllerManagerImageOpenStack: "registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.31.1@sha256:72cc0d22b83c613df809d8134e50404171513d92287e63e2313d9ad7e1ed630e", // renovate:container + CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.31.0@sha256:6d4c51c35f344d230341d71bb6d35f2c2f0c0a6f205a7887ae44e6d852fb5b5f", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.29.4@sha256:786728c85787a58c6376b47d2e22cc04db3ecfdd73a52b5b9be20fd869abce2f", // renovate:container }, - V1_29: { - ClusterVersion: "v1.29.8", // renovate:kubernetes-release + V1_30: { + ClusterVersion: "v1.30.4", // renovate:kubernetes-release KubernetesComponents: components.Components{ { Url: "https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz", // renovate:cni-plugins-release @@ -274,39 +274,39 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.30.0/crictl-v1.30.0-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:3dd03954565808eaeb3a7ffc0e8cb7886a64a9aa94b2bfdfbdc6e2ed94842e49", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:df6e130928403af8b4f49f1197e26f2873a147cd0e23aa6597a26c982c652ae0", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:0c02c0f997b3e9769eae7ca051856054411fca947b3d5409d991ce1964dd0e69", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:fe054355e0ae8dc35d868a3d3bc408ccdff0969c20bf7a231ae9b71484e41be3", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:6c6053fb8b31030ef7fffe146eb29489f7bf53d7a5ca10e0b10c907bf4b7e281", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:038454e0d79748aab41668f44ca6e4ac8affd1895a94f592b9739a0ae2a5f06a", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:2ffd023712bbc1a9390dbd8c0c15201c165a69d394787ef03eda3eccb4b9ac06", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI5LjhAc2hhMjU2OjZmNzJmYTkyNmM5YjA1ZTEwNjI5ZmUxYTA5MmZkMjhkY2Q2NWI0ZmRmZDBjYzdiZDU1Zjg1YTU3YTZiYTFmYTUifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMwLjRAc2hhMjU2OjdiMGM0YTk1OWFhZWU1NjYwZTEyMzQ0NTJkYzMxMjMzMTAyMzFiOWY5MmQyOWViZDE3NWM4NmRjOWY3OTdlZTcifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI5LjhAc2hhMjU2OjZmMjdkNjNkZWQyMDYxNGM2ODU1NGI0NzdjZDdhNzhlZGE3OGE0OThhOTJiZmU4OTM1Y2Y5NjRjYTViNzRkMGIifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMwLjRAc2hhMjU2Ojk5MmNjY2JmNjUyZmE5NTFjMWEzZDQxYjBjMTAzM2FlMGJmNjRmMzNkYTAzZDUwMzk1MjgyYzU1MTkwMGFmOWUifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI5LjhAc2hhMjU2OmRhNzRhNjY2NzVkOTVlMzllYzI1ZGE1ZTcwNzI5ZGE3NDZkMGZhMGIxNWVlMGRhODcyYWM5ODA1MTliYzI4YmQifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMwLjRAc2hhMjU2OjM3ZWFlZWU1YmNhOGRhMzRhZDNkMzZlMzc1ODZkZDI5ZjVlZGIxZTI5MjdlNzY0NGRmYjExM2U3MDA2MmJkYTgifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -316,64 +316,64 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.29.6@sha256:8074b8828a33fb273833e8fd374dda6a0ab10335ae8e19684fbd61eeff7d3594", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.3@sha256:30a1758dec30814178c787e2d50f46bb141e9f0bb2e16190ddd19df15f957874", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.29.10@sha256:2e4d2c0bb15f2987cb7abb52e86c9b62504226fdbf19d5b597ac4707adb52a56", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.30.6@sha256:4cbf44b038ff426dccb457b160f7c675d736a062ec9e729996c568bf3d252c67", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.29.10@sha256:ef27f4e2c104614fa44265d6e0a193bf1f9b5abda542f5b64efb868d3796ffc8", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.30.6@sha256:89ea8b66d026fe313f7f5781d1be6c44bf2c345ead89b8fe256c2bfab47304d1", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. - CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v29.5.1@sha256:ebbc6f5755725b6c2c81ca1d1580e2feba83572c41608b739c50f85b2e5de936", // renovate:container + CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.0@sha256:64d2d5d4d2b5fb426c307c64ada9a61b64e797b56d9768363f145f2bd957998f", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.29.4@sha256:786728c85787a58c6376b47d2e22cc04db3ecfdd73a52b5b9be20fd869abce2f", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.2@sha256:ef370d1d06a1603c4b1e47e64bb27c3ff51cb20680712dc3d41c34f3fbf7be9f", // renovate:container }, - V1_30: { - ClusterVersion: "v1.30.4", // renovate:kubernetes-release + V1_31: { + ClusterVersion: "v1.31.1", // renovate:kubernetes-release KubernetesComponents: components.Components{ { - Url: "https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz", // renovate:cni-plugins-release - Hash: "sha256:c2485ddb3ffc176578ae30ae58137f0b88e50f7c7f2af7d53a569276b2949a33", + Url: "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz", // renovate:cni-plugins-release + Hash: "sha256:77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b", InstallPath: constants.CniPluginsDir, Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.30.0/crictl-v1.30.0-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:3dd03954565808eaeb3a7ffc0e8cb7886a64a9aa94b2bfdfbdc6e2ed94842e49", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:0a03ba6b1e4c253d63627f8d210b2ea07675a8712587e697657b236d06d7d231", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:0c02c0f997b3e9769eae7ca051856054411fca947b3d5409d991ce1964dd0e69", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:50619fff95bdd7e690c049cc083f495ae0e7c66d0cdf6a8bcad298af5fe28438", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:6c6053fb8b31030ef7fffe146eb29489f7bf53d7a5ca10e0b10c907bf4b7e281", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:b3f92d19d482359116dd9ee9c0a10cb86e32a2a2aef79b853d5f07d6a093b0df", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:2ffd023712bbc1a9390dbd8c0c15201c165a69d394787ef03eda3eccb4b9ac06", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:57b514a7facce4ee62c93b8dc21fda8cf62ef3fed22e44ffc9d167eab843b2ae", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMwLjRAc2hhMjU2OjdiMGM0YTk1OWFhZWU1NjYwZTEyMzQ0NTJkYzMxMjMzMTAyMzFiOWY5MmQyOWViZDE3NWM4NmRjOWY3OTdlZTcifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMxLjFAc2hhMjU2OjI0MDljMjNkYmI1YTJiN2E4MWFkYmIxODRkM2VhYzQzYWM2NTNlOWI5N2E3YzBlZTEyMWI4OWJiM2VmNjFmZGIifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMwLjRAc2hhMjU2Ojk5MmNjY2JmNjUyZmE5NTFjMWEzZDQxYjBjMTAzM2FlMGJmNjRmMzNkYTAzZDUwMzk1MjgyYzU1MTkwMGFmOWUifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMxLjFAc2hhMjU2OjlmOWRhNWIyN2UwM2Y4OTU5OWNjNDBiYTg5MTUwYWViZjNiNGNmZjAwMWU2ZGI2ZDk5ODY3NGIzNDE4MWUxYTEifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMwLjRAc2hhMjU2OjM3ZWFlZWU1YmNhOGRhMzRhZDNkMzZlMzc1ODZkZDI5ZjVlZGIxZTI5MjdlNzY0NGRmYjExM2U3MDA2MmJkYTgifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMxLjFAc2hhMjU2Ojk2OWE3ZTk2MzQwZjNhOTI3YjNkNjUyNTgyZWRlYzJkNmQ4MmEwODM4NzFkODFlZjUwNjRiN2VkYWFiNDMwZDAifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -383,20 +383,20 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.3@sha256:30a1758dec30814178c787e2d50f46bb141e9f0bb2e16190ddd19df15f957874", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.31.0@sha256:c4658f5c583e85ef58a096ac9901a8ced3cb9699d8eeceb59d3bb54c5d0db72f", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.30.6@sha256:4cbf44b038ff426dccb457b160f7c675d736a062ec9e729996c568bf3d252c67", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.31.1@sha256:b5aa55a7e9d38137f7fcd0adc9335b06e7c96061764addd7e6bb9f86403f0110", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.30.6@sha256:89ea8b66d026fe313f7f5781d1be6c44bf2c345ead89b8fe256c2bfab47304d1", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.1@sha256:e9b522399e4ec6bc4ce90c173e59db135d742de7b16f0f5454b4d88ba78a98c7", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.0@sha256:64d2d5d4d2b5fb426c307c64ada9a61b64e797b56d9768363f145f2bd957998f", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. - CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container + CloudControllerManagerImageOpenStack: "registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.31.1@sha256:72cc0d22b83c613df809d8134e50404171513d92287e63e2313d9ad7e1ed630e", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.2@sha256:ef370d1d06a1603c4b1e47e64bb27c3ff51cb20680712dc3d41c34f3fbf7be9f", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.31.0@sha256:6d4c51c35f344d230341d71bb6d35f2c2f0c0a6f205a7887ae44e6d852fb5b5f", // renovate:container }, } From c70bdc48d9f645215c48138dcfcb2f0bd27eaac6 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 9 Oct 2024 15:25:03 +0200 Subject: [PATCH 05/15] e2e: run vale with python venv --- .github/workflows/docs-vale.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs-vale.yml b/.github/workflows/docs-vale.yml index 505b291784..a1cfc4cbd9 100644 --- a/.github/workflows/docs-vale.yml +++ b/.github/workflows/docs-vale.yml @@ -19,7 +19,11 @@ jobs: uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - + # Work around https://github.com/errata-ai/vale-action/issues/128. + - run: | + venv="$HOME/.local/share/venv" + python3 -m venv "$venv" + echo "$venv/bin" >> "$GITHUB_PATH" - name: Vale uses: errata-ai/vale-action@91ac403e8d26f5aa1b3feaa86ca63065936a85b6 # tag=reviewdog with: From 91b64351c1d371d28ffd79c157a3dc66c34b35c0 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Thu, 10 Oct 2024 20:04:08 +0200 Subject: [PATCH 06/15] wip: misc: set feature gate ControlPlaneKubeletLocalMode --- .../kubernetes/k8sapi/kubeadm_config.go | 3 + internal/constellation/kubecmd/kubecmd.go | 132 +++++++++++------- 2 files changed, 81 insertions(+), 54 deletions(-) diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go index 430839a469..b494b73fe1 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go @@ -122,6 +122,9 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl "profiling": "false", }, }, + FeatureGates: map[string]bool{ + "ControlPlaneKubeletLocalMode": true, + }, }, // warning: this config is applied to every node in the cluster! KubeletConfiguration: kubeletconf.KubeletConfiguration{ diff --git a/internal/constellation/kubecmd/kubecmd.go b/internal/constellation/kubecmd/kubecmd.go index a193f2528e..dd3c4f9da7 100644 --- a/internal/constellation/kubecmd/kubecmd.go +++ b/internal/constellation/kubecmd/kubecmd.go @@ -131,6 +131,18 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio ) } + // TODO(burgerdev): remove after releasing v2.19 + // Workaround for https://github.com/kubernetes/kubernetes/issues/127316: force kubelet to + // connect to the local API server. + if err := k.patchKubeadmConfig(ctx, func(cc *kubeadm.ClusterConfiguration) { + if cc.FeatureGates == nil { + cc.FeatureGates = map[string]bool{} + } + cc.FeatureGates["ControlPlaneKubeletLocalMode"] = true + }); err != nil { + return fmt.Errorf("setting FeatureGate ControlPlaneKubeletLocalMode: %w", err) + } + versionConfig, ok := versions.VersionConfigs[kubernetesVersion] if !ok { return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError( @@ -236,65 +248,32 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At // ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs. // Empty strings are ignored, existing SANs are preserved. func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error { - var kubeadmConfig *corev1.ConfigMap - if err := k.retryAction(ctx, func(ctx context.Context) error { - var err error - kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) - return err - }); err != nil { - return fmt.Errorf("retrieving current kubeadm-config: %w", err) - } - - clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] - if !ok { - return errors.New("ClusterConfiguration missing from kubeadm-config") - } - - var clusterConfiguration kubeadm.ClusterConfiguration - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { - return fmt.Errorf("decoding cluster configuration data: %w", err) - } - - existingSANs := make(map[string]struct{}) - for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { - existingSANs[existingSAN] = struct{}{} - } - - var missingSANs []string - for _, san := range alternativeNames { - if san == "" { - continue // skip empty SANs + if err := k.patchKubeadmConfig(ctx, func(clusterConfiguration *kubeadm.ClusterConfiguration) { + existingSANs := make(map[string]struct{}) + for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { + existingSANs[existingSAN] = struct{}{} } - if _, ok := existingSANs[san]; !ok { - missingSANs = append(missingSANs, san) - existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice - } - } - if len(missingSANs) == 0 { - k.log.Debug("No new SANs to add to the cluster's apiserver SAN field") - return nil - } - k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", ")) - - clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) - sort.Strings(clusterConfiguration.APIServer.CertSANs) + var missingSANs []string + for _, san := range alternativeNames { + if san == "" { + continue // skip empty SANs + } + if _, ok := existingSANs[san]; !ok { + missingSANs = append(missingSANs, san) + existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice + } + } - opt := k8sjson.SerializerOptions{Yaml: true} - serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) - encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) - newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) - if err != nil { - return fmt.Errorf("marshaling ClusterConfiguration: %w", err) - } + if len(missingSANs) == 0 { + k.log.Debug("No new SANs to add to the cluster's apiserver SAN field") + } + k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", ")) - kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) - k.log.Debug("Triggering kubeadm config update now") - if err = k.retryAction(ctx, func(ctx context.Context) error { - _, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig) - return err + clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) + sort.Strings(clusterConfiguration.APIServer.CertSANs) }); err != nil { - return fmt.Errorf("setting new kubeadm config: %w", err) + return fmt.Errorf("extending ClusterConfig.CertSANs: %w", err) } k.log.Debug("Successfully extended the cluster's apiserver SAN field") @@ -462,6 +441,51 @@ func (k *KubeCmd) retryAction(ctx context.Context, action func(ctx context.Conte return retrier.Do(ctx) } +// patchKubeadmConfig fetches and unpacks the kube-system/kubeadm-config ClusterConfiguration entry, +// runs doPatch on it and uploads the result. +func (k *KubeCmd) patchKubeadmConfig(ctx context.Context, doPatch func(*kubeadm.ClusterConfiguration)) error { + var kubeadmConfig *corev1.ConfigMap + if err := k.retryAction(ctx, func(ctx context.Context) error { + var err error + kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) + return err + }); err != nil { + return fmt.Errorf("retrieving current kubeadm-config: %w", err) + } + + clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] + if !ok { + return errors.New("ClusterConfiguration missing from kubeadm-config") + } + + var clusterConfiguration kubeadm.ClusterConfiguration + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { + return fmt.Errorf("decoding cluster configuration data: %w", err) + } + + doPatch(&clusterConfiguration) + + opt := k8sjson.SerializerOptions{Yaml: true} + serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) + encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) + newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) + if err != nil { + return fmt.Errorf("marshaling ClusterConfiguration: %w", err) + } + + kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) + k.log.Debug("Triggering kubeadm config update now") + if err = k.retryAction(ctx, func(ctx context.Context) error { + _, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig) + return err + }); err != nil { + return fmt.Errorf("setting new kubeadm config: %w", err) + } + + k.log.Debug("Successfully patched the cluster's kubeadm-config") + return nil +} + func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error { var err error switch { From d5bb722f794874894768341dd27e020d9f64031a Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Sun, 20 Oct 2024 22:06:15 +0200 Subject: [PATCH 07/15] kubernetes: fixup feature flag Only use feature flag on K8s versions that support that feature flag. Otherwise Constellation throws an error during init. Also fixup some tests. --- bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel | 1 + .../internal/kubernetes/k8sapi/kubeadm_config.go | 11 +++++++---- internal/constellation/kubecmd/kubecmd_test.go | 3 +++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel index ef87085bd8..968a2b0114 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel +++ b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "@io_k8s_kubelet//config/v1beta1", "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@io_k8s_kubernetes//cmd/kubeadm/app/constants", + "@org_golang_x_mod//semver", ], ) diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go index b494b73fe1..1ce387aeb4 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go @@ -12,6 +12,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/kubernetes" + "golang.org/x/mod/semver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconf "k8s.io/kubelet/config/v1beta1" @@ -38,7 +39,7 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl cloudProvider = "external" } - return KubeadmInitYAML{ + initConfig := KubeadmInitYAML{ InitConfiguration: kubeadm.InitConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), @@ -122,9 +123,6 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl "profiling": "false", }, }, - FeatureGates: map[string]bool{ - "ControlPlaneKubeletLocalMode": true, - }, }, // warning: this config is applied to every node in the cluster! KubeletConfiguration: kubeletconf.KubeletConfiguration{ @@ -160,6 +158,11 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl TLSPrivateKeyFile: certificate.KeyFilename, }, } + + if semver.Compare(clusterVersion, "v1.31.0") >= 0 { + initConfig.ClusterConfiguration.FeatureGates = map[string]bool{"ControlPlaneKubeletLocalMode": true} + } + return initConfig } // JoinConfiguration returns a new kubeadm join configuration. diff --git a/internal/constellation/kubecmd/kubecmd_test.go b/internal/constellation/kubecmd/kubecmd_test.go index 79eef03c34..c88dab9bda 100644 --- a/internal/constellation/kubecmd/kubecmd_test.go +++ b/internal/constellation/kubecmd/kubecmd_test.go @@ -281,6 +281,9 @@ func TestUpgradeKubernetesVersion(t *testing.T) { } kubectl := &stubKubectl{ unstructuredInterface: unstructuredClient, + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": kubeadmClusterConfigurationV1Beta4}}, + }, } if tc.customClientFn != nil { kubectl.unstructuredInterface = tc.customClientFn(nodeVersion) From 01af37a44962ba1bc4c04edc53bb56d23eec0770 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Sun, 20 Oct 2024 23:17:48 +0200 Subject: [PATCH 08/15] operator: bump controller-gen version This bump the controller gen version and also adjusts the generate commands (back to the original ones). This allows correct generation of CRDs and go code. --- .../crds/autoscalingstrategy-crd.yaml | 35 +- .../crds/joiningnode-crd.yaml | 28 +- .../crds/nodeversion-crd.yaml | 681 +++++++----------- .../crds/pendingnode-crd.yaml | 44 +- .../crds/scalinggroup-crd.yaml | 98 ++- .../templates/manager-rbac.yaml | 114 +-- .../templates/manager-rbac.yaml | 114 +-- .../templates/manager-rbac.yaml | 114 +-- .../templates/manager-rbac.yaml | 114 +-- .../templates/manager-rbac.yaml | 114 +-- .../templates/manager-rbac.yaml | 114 +-- .../constellation-node-operator/Makefile | 6 +- .../api/v1alpha1/zz_generated.deepcopy.go | 1 - ...dgeless.systems_autoscalingstrategies.yaml | 20 +- .../update.edgeless.systems_joiningnodes.yaml | 20 +- .../update.edgeless.systems_nodeversions.yaml | 678 +++++++---------- .../update.edgeless.systems_pendingnodes.yaml | 29 +- ...update.edgeless.systems_scalinggroups.yaml | 70 +- .../config/rbac/role.yaml | 115 +-- 19 files changed, 779 insertions(+), 1730 deletions(-) diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml index 18dce5e376..9156e3e714 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: autoscalingstrategies.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: autoscalingstrategies.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -20,14 +21,19 @@ spec: API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -48,8 +54,8 @@ spec: deployment. type: string enabled: - description: Enabled defines whether cluster autoscaling should be enabled - or not. + description: Enabled defines whether cluster autoscaling should be + enabled or not. type: boolean required: - deploymentName @@ -64,7 +70,8 @@ spec: enabled or not. type: boolean replicas: - description: Replicas is the number of replicas for the autoscaler deployment. + description: Replicas is the number of replicas for the autoscaler + deployment. format: int32 type: integer type: object @@ -73,9 +80,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml index 88fb65ae81..1beca72211 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: joiningnodes.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: joiningnodes.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: JoiningNode is the Schema for the joiningnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -59,9 +65,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml index 9c46b695cd..4b7f7b7e04 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: nodeversions.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: nodeversions.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: NodeVersion is the Schema for the nodeversions API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -60,65 +66,49 @@ spec: description: AwaitingAnnotation is a list of nodes that are waiting for the operator to annotate them. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array budget: description: Budget is the amount of extra nodes that can be created @@ -129,43 +119,35 @@ spec: description: Conditions represent the latest available observations of an object's state items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 @@ -180,10 +162,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -199,516 +177,389 @@ spec: description: Donors is a list of outdated nodes that donate labels to heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array heirs: description: Heirs is a list of nodes using the latest image that still need to inherit labels from donors. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array invalid: description: Invalid is a list of invalid nodes (nodes that cannot be processed by the operator due to missing information or transient faults). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array mints: description: Mints is a list of up to date nodes that will become heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array obsolete: description: Obsolete is a list of obsolete nodes (nodes that have been created by the operator but are no longer needed). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array outdated: description: Outdated is a list of nodes that are using an outdated image. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array pending: description: Pending is a list of pending nodes (joining or leaving the cluster). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array upToDate: description: UpToDate is a list of nodes that are using the latest image and labels. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array required: + - activeclusterversionupgrade - budget - conditions type: object diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml index 41b5a4cd72..c6cd2db6ac 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: pendingnodes.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: pendingnodes.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: PendingNode is the Schema for the pendingnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -34,10 +40,11 @@ spec: description: PendingNodeSpec defines the desired state of PendingNode. properties: deadline: - description: Deadline is the deadline for reaching the goal state. Joining - nodes will be terminated if the deadline is exceeded. Leaving nodes - will remain as unschedulable to prevent data loss. If not specified, - the node may remain in the pending state indefinitely. + description: |- + Deadline is the deadline for reaching the goal state. + Joining nodes will be terminated if the deadline is exceeded. + Leaving nodes will remain as unschedulable to prevent data loss. + If not specified, the node may remain in the pending state indefinitely. format: date-time type: string goal: @@ -47,8 +54,8 @@ spec: - Leave type: string groupID: - description: ScalingGroupID is the ID of the group that this node shall - be part of. + description: ScalingGroupID is the ID of the group that this node + shall be part of. type: string nodeName: description: NodeName is the kubernetes internal name of the node. @@ -72,7 +79,8 @@ spec: - Failed type: string reachedGoal: - description: ReachedGoal is true if the node has reached the goal state. + description: ReachedGoal is true if the node has reached the goal + state. type: boolean type: object type: object @@ -80,9 +88,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml index 0e334ae297..5eed4ebc88 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: scalinggroups.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: scalinggroups.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: ScalingGroup is the Schema for the scalinggroups API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -37,16 +43,16 @@ spec: description: AutoscalerGroupName is name that is expected by the autoscaler. type: string autoscaling: - description: Autoscaling specifies wether the scaling group should automatically - scale using the cluster-autoscaler. + description: Autoscaling specifies wether the scaling group should + automatically scale using the cluster-autoscaler. type: boolean groupId: - description: GroupID is the CSP specific, canonical identifier of a - scaling group. + description: GroupID is the CSP specific, canonical identifier of + a scaling group. type: string max: - description: Max is the maximum number of autoscaled nodes in the scaling - group (used by cluster-autoscaler). + description: Max is the maximum number of autoscaled nodes in the + scaling group (used by cluster-autoscaler). format: int32 type: integer min: @@ -55,11 +61,11 @@ spec: format: int32 type: integer nodeGroupName: - description: NodeGroupName is the human friendly name of the node group - as defined in the Constellation configuration. + description: NodeGroupName is the human friendly name of the node + group as defined in the Constellation configuration. type: string nodeImage: - description: NodeImage is the name of the NodeImage resource. + description: NodeVersion is the name of the NodeVersion resource. type: string role: description: Role is the role of the nodes in the scaling group. @@ -75,44 +81,36 @@ spec: description: Conditions represent the latest available observations of an object's state. items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a foo's - current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating details - about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers of - specific condition types may define expected values and meanings - for this field, and whether the values are considered a guaranteed - API. The value should be a CamelCase string. This field may - not be empty. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. maxLength: 1024 minLength: 1 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ @@ -126,10 +124,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -142,8 +136,8 @@ spec: type: object type: array imageReference: - description: ImageReference is the image currently used for newly created - nodes in this scaling group. + description: ImageReference is the image currently used for newly + created nodes in this scaling group. type: string required: - conditions @@ -153,9 +147,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml index 45dddbdd92..b75dfed440 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -58,98 +58,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -161,13 +73,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -175,29 +95,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/operators/constellation-node-operator/Makefile b/operators/constellation-node-operator/Makefile index ca2bd09068..777b1ecb1a 100644 --- a/operators/constellation-node-operator/Makefile +++ b/operators/constellation-node-operator/Makefile @@ -86,11 +86,11 @@ help: ## Display this help. .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./" output:crd:artifacts:config=config/crd/bases + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./" + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." .PHONY: fmt fmt: ## Run go fmt against code. @@ -162,7 +162,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ## Tool Versions KUSTOMIZE_VERSION ?= v3.8.7 -CONTROLLER_TOOLS_VERSION ?= v0.9.0 +CONTROLLER_TOOLS_VERSION ?= v0.16.4 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize diff --git a/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go b/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go index 198fc055ce..b25cfc280f 100644 --- a/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_autoscalingstrategies.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_autoscalingstrategies.yaml index 6a014a3947..9156e3e714 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_autoscalingstrategies.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_autoscalingstrategies.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: autoscalingstrategies.update.edgeless.systems spec: group: update.edgeless.systems @@ -22,14 +21,19 @@ spec: API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml index 9c8e9e5a29..1beca72211 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: joiningnodes.update.edgeless.systems spec: group: update.edgeless.systems @@ -21,14 +20,19 @@ spec: description: JoiningNode is the Schema for the joiningnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml index e4c435ec1c..4b7f7b7e04 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: nodeversions.update.edgeless.systems spec: group: update.edgeless.systems @@ -21,14 +20,19 @@ spec: description: NodeVersion is the Schema for the nodeversions API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -62,65 +66,49 @@ spec: description: AwaitingAnnotation is a list of nodes that are waiting for the operator to annotate them. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array budget: description: Budget is the amount of extra nodes that can be created @@ -131,43 +119,35 @@ spec: description: Conditions represent the latest available observations of an object's state items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 @@ -182,10 +162,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -201,514 +177,386 @@ spec: description: Donors is a list of outdated nodes that donate labels to heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array heirs: description: Heirs is a list of nodes using the latest image that still need to inherit labels from donors. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array invalid: description: Invalid is a list of invalid nodes (nodes that cannot be processed by the operator due to missing information or transient faults). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array mints: description: Mints is a list of up to date nodes that will become heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array obsolete: description: Obsolete is a list of obsolete nodes (nodes that have been created by the operator but are no longer needed). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array outdated: description: Outdated is a list of nodes that are using an outdated image. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array pending: description: Pending is a list of pending nodes (joining or leaving the cluster). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array upToDate: description: UpToDate is a list of nodes that are using the latest image and labels. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array required: - activeclusterversionupgrade diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_pendingnodes.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_pendingnodes.yaml index 7c5b5618b5..c6cd2db6ac 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_pendingnodes.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_pendingnodes.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: pendingnodes.update.edgeless.systems spec: group: update.edgeless.systems @@ -21,14 +20,19 @@ spec: description: PendingNode is the Schema for the pendingnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -36,10 +40,11 @@ spec: description: PendingNodeSpec defines the desired state of PendingNode. properties: deadline: - description: Deadline is the deadline for reaching the goal state. - Joining nodes will be terminated if the deadline is exceeded. Leaving - nodes will remain as unschedulable to prevent data loss. If not - specified, the node may remain in the pending state indefinitely. + description: |- + Deadline is the deadline for reaching the goal state. + Joining nodes will be terminated if the deadline is exceeded. + Leaving nodes will remain as unschedulable to prevent data loss. + If not specified, the node may remain in the pending state indefinitely. format: date-time type: string goal: diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_scalinggroups.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_scalinggroups.yaml index 0f87fbab1c..5eed4ebc88 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_scalinggroups.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_scalinggroups.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: scalinggroups.update.edgeless.systems spec: group: update.edgeless.systems @@ -21,14 +20,19 @@ spec: description: ScalingGroup is the Schema for the scalinggroups API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -57,8 +61,8 @@ spec: format: int32 type: integer nodeGroupName: - description: NodeGroupName is the human friendly name of the node group - as defined in the Constellation configuration. + description: NodeGroupName is the human friendly name of the node + group as defined in the Constellation configuration. type: string nodeImage: description: NodeVersion is the name of the NodeVersion resource. @@ -77,43 +81,35 @@ spec: description: Conditions represent the latest available observations of an object's state. items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 @@ -128,10 +124,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string diff --git a/operators/constellation-node-operator/config/rbac/role.yaml b/operators/constellation-node-operator/config/rbac/role.yaml index 8700132c49..169011983a 100644 --- a/operators/constellation-node-operator/config/rbac/role.yaml +++ b/operators/constellation-node-operator/config/rbac/role.yaml @@ -2,7 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: manager-role rules: - apiGroups: @@ -57,98 +56,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -160,13 +71,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -174,26 +93,14 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update From 15d7de5883b7ef2e547011e4e047d4f8cd81101d Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Sun, 20 Oct 2024 22:12:56 +0200 Subject: [PATCH 09/15] operator: allow increasing number of node upgrade budget --- .../crds/nodeversion-crd.yaml | 5 +++++ .../api/v1alpha1/nodeversion_types.go | 2 ++ .../update.edgeless.systems_nodeversions.yaml | 5 +++++ .../config/manager/kustomization.yaml | 2 +- .../controllers/nodeversion_controller.go | 17 ++++++++++------- .../controllers/nodeversion_controller_test.go | 2 +- 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml index 4b7f7b7e04..04693de2e8 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml @@ -54,6 +54,11 @@ spec: description: KubernetesComponentsReference is a reference to the ConfigMap containing the Kubernetes components to use for all nodes. type: string + maxNodeBudget: + description: MaxNodeBudget is the maximum number of nodes that can + be created simultaneously. + format: int32 + type: integer type: object status: description: NodeVersionStatus defines the observed state of NodeVersion. diff --git a/operators/constellation-node-operator/api/v1alpha1/nodeversion_types.go b/operators/constellation-node-operator/api/v1alpha1/nodeversion_types.go index f5d81dfede..6e741c7d83 100644 --- a/operators/constellation-node-operator/api/v1alpha1/nodeversion_types.go +++ b/operators/constellation-node-operator/api/v1alpha1/nodeversion_types.go @@ -21,6 +21,8 @@ type NodeVersionSpec struct { KubernetesComponentsReference string `json:"kubernetesComponentsReference,omitempty"` // KubernetesClusterVersion is the advertised Kubernetes version of the cluster. KubernetesClusterVersion string `json:"kubernetesClusterVersion,omitempty"` + // MaxNodeBudget is the maximum number of nodes that can be created simultaneously. + MaxNodeBudget uint32 `json:"maxNodeBudget,omitempty"` } // NodeVersionStatus defines the observed state of NodeVersion. diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml index 4b7f7b7e04..04693de2e8 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml @@ -54,6 +54,11 @@ spec: description: KubernetesComponentsReference is a reference to the ConfigMap containing the Kubernetes components to use for all nodes. type: string + maxNodeBudget: + description: MaxNodeBudget is the maximum number of nodes that can + be created simultaneously. + format: int32 + type: integer type: object status: description: NodeVersionStatus defines the observed state of NodeVersion. diff --git a/operators/constellation-node-operator/config/manager/kustomization.yaml b/operators/constellation-node-operator/config/manager/kustomization.yaml index 23ff3a7e4c..6919d782b9 100644 --- a/operators/constellation-node-operator/config/manager/kustomization.yaml +++ b/operators/constellation-node-operator/config/manager/kustomization.yaml @@ -13,4 +13,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/edgelesssys/constellation/node-operator - newTag: v0.0.0 + newTag: v0.0.1 diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller.go b/operators/constellation-node-operator/controllers/nodeversion_controller.go index ff706c702f..cfa3e315e2 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller.go @@ -38,8 +38,6 @@ import ( ) const ( - // nodeOverprovisionLimit is the maximum number of extra nodes created during the update procedure at any point in time. - nodeOverprovisionLimit = 1 // nodeJoinTimeout is the time limit pending nodes have to join the cluster before being terminated. nodeJoinTimeout = time.Minute * 30 // nodeLeaveTimeout is the time limit pending nodes have to leave the cluster and being terminated. @@ -161,9 +159,14 @@ func (r *NodeVersionReconciler) Reconcile(ctx context.Context, req ctrl.Request) // - being created (joining) // - being destroyed (leaving) // - heirs to outdated nodes - extraNodes := len(groups.Heirs) + len(groups.AwaitingAnnotation) + len(pendingNodeList.Items) + extraNodes := uint32(len(groups.Heirs) + len(groups.AwaitingAnnotation) + len(pendingNodeList.Items)) // newNodesBudget is the maximum number of new nodes that can be created in this Reconcile call. - var newNodesBudget int + var newNodesBudget uint32 + nodeOverprovisionLimit := uint32(1) + // Use user definable value if it makes sense + if desiredNodeVersion.Spec.MaxNodeBudget > 0 { + nodeOverprovisionLimit = desiredNodeVersion.Spec.MaxNodeBudget + } if extraNodes < nodeOverprovisionLimit { newNodesBudget = nodeOverprovisionLimit - extraNodes } @@ -743,7 +746,7 @@ func (r *NodeVersionReconciler) tryUpdateStatus(ctx context.Context, name types. } // nodeVersionStatus generates the NodeVersion.Status field given node groups and the budget for new nodes. -func nodeVersionStatus(scheme *runtime.Scheme, groups nodeGroups, pendingNodes []updatev1alpha1.PendingNode, invalidNodes []corev1.Node, newNodesBudget int) updatev1alpha1.NodeVersionStatus { +func nodeVersionStatus(scheme *runtime.Scheme, groups nodeGroups, pendingNodes []updatev1alpha1.PendingNode, invalidNodes []corev1.Node, newNodesBudget uint32) updatev1alpha1.NodeVersionStatus { var status updatev1alpha1.NodeVersionStatus outdatedCondition := metav1.Condition{ Type: updatev1alpha1.ConditionOutdated, @@ -821,7 +824,7 @@ func nodeVersionStatus(scheme *runtime.Scheme, groups nodeGroups, pendingNodes [ } status.Pending = append(status.Pending, *pendingRef) } - status.Budget = uint32(newNodesBudget) + status.Budget = newNodesBudget return status } @@ -941,5 +944,5 @@ type newNodeConfig struct { outdatedNodes []corev1.Node pendingNodes []updatev1alpha1.PendingNode scalingGroupByID map[string]updatev1alpha1.ScalingGroup - newNodesBudget int + newNodesBudget uint32 } diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go index c9ae880421..e32b99af3d 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go @@ -332,7 +332,7 @@ func TestCreateNewNodes(t *testing.T) { outdatedNodes []corev1.Node pendingNodes []updatev1alpha1.PendingNode scalingGroupByID map[string]updatev1alpha1.ScalingGroup - budget int + budget uint32 wantCreateCalls []string }{ "no outdated nodes": { From 5a59d102723916ec7f717398ff93a78dadf31fa9 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Sun, 20 Oct 2024 22:14:13 +0200 Subject: [PATCH 10/15] operator: requeue scaling group on conflict --- .../controllers/scalinggroup_controller.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/operators/constellation-node-operator/controllers/scalinggroup_controller.go b/operators/constellation-node-operator/controllers/scalinggroup_controller.go index 7336467c0a..35743f3cc7 100644 --- a/operators/constellation-node-operator/controllers/scalinggroup_controller.go +++ b/operators/constellation-node-operator/controllers/scalinggroup_controller.go @@ -10,6 +10,7 @@ import ( "context" "strings" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -92,8 +93,10 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request } meta.SetStatusCondition(&desiredScalingGroup.Status.Conditions, outdatedCondition) if err := r.Status().Update(ctx, &desiredScalingGroup); err != nil { - logr.Error(err, "Unable to update AutoscalingStrategy status") - return ctrl.Result{}, err + if !k8sErrors.IsConflict(err) { + logr.Error(err, "Unable to update Scaling Group status") + } + return ctrl.Result{Requeue: true}, err } if !imagesMatch { From 49ef340fc3b098dad8e5bbd949e6a2c0858848dd Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Sun, 20 Oct 2024 22:14:32 +0200 Subject: [PATCH 11/15] operator: clean up after joining node env test --- .../controllers/joiningnode_controller_env_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go b/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go index 2662dff284..4dfa2907b4 100644 --- a/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go +++ b/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go @@ -39,6 +39,13 @@ var _ = Describe("JoiningNode controller", func() { duration = time.Second * 2 interval = time.Millisecond * 250 ) + + AfterEach(func() { + Eventually(func() error { + return resetEnv() + }, 30*time.Second, 1*time.Second).Should(Succeed()) + }) + Context("When changing a joining node resource spec", func() { It("Should annotate the corresponding node when creating the CRD first", func() { By("creating a joining node resource") From ce2e48c82a692a87c320368abdc530671649f8b7 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Sun, 20 Oct 2024 22:16:47 +0200 Subject: [PATCH 12/15] operator: move control plane label to constants --- .../constellation-node-operator/controllers/BUILD.bazel | 1 + .../internal/constants/constants.go | 2 ++ .../internal/controlplane/BUILD.bazel | 1 + .../internal/controlplane/controlplane_test.go | 3 ++- .../constellation-node-operator/internal/etcd/BUILD.bazel | 1 + .../constellation-node-operator/internal/etcd/etcd_test.go | 3 ++- .../constellation-node-operator/internal/node/BUILD.bazel | 2 ++ operators/constellation-node-operator/internal/node/node.go | 5 ++--- .../constellation-node-operator/internal/node/node_test.go | 3 ++- 9 files changed, 15 insertions(+), 6 deletions(-) diff --git a/operators/constellation-node-operator/controllers/BUILD.bazel b/operators/constellation-node-operator/controllers/BUILD.bazel index f665358091..2a24c2584b 100644 --- a/operators/constellation-node-operator/controllers/BUILD.bazel +++ b/operators/constellation-node-operator/controllers/BUILD.bazel @@ -81,6 +81,7 @@ go_test( "//3rdparty/node-maintenance-operator/api/v1beta1", "//internal/constants", "//operators/constellation-node-operator/api/v1alpha1", + "//operators/constellation-node-operator/internal/constants", "@com_github_onsi_ginkgo_v2//:ginkgo", "@com_github_onsi_gomega//:gomega", "@com_github_stretchr_testify//assert", diff --git a/operators/constellation-node-operator/internal/constants/constants.go b/operators/constellation-node-operator/internal/constants/constants.go index b057df59cb..db49353003 100644 --- a/operators/constellation-node-operator/internal/constants/constants.go +++ b/operators/constellation-node-operator/internal/constants/constants.go @@ -19,4 +19,6 @@ const ( PlaceholderControlPlaneScalingGroupName = "control-planes-id" // PlaceholderWorkerScalingGroupName name of the worker scaling group used if upgrades are not yet supported. PlaceholderWorkerScalingGroupName = "workers-id" + // ControlPlaneRoleLabel label used to identify control plane nodes. + ControlPlaneRoleLabel = "node-role.kubernetes.io/control-plane" ) diff --git a/operators/constellation-node-operator/internal/controlplane/BUILD.bazel b/operators/constellation-node-operator/internal/controlplane/BUILD.bazel index 65651541d0..c2495adbb4 100644 --- a/operators/constellation-node-operator/internal/controlplane/BUILD.bazel +++ b/operators/constellation-node-operator/internal/controlplane/BUILD.bazel @@ -18,6 +18,7 @@ go_test( srcs = ["controlplane_test.go"], embed = [":controlplane"], deps = [ + "//operators/constellation-node-operator/internal/constants", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@io_k8s_api//core/v1:core", diff --git a/operators/constellation-node-operator/internal/controlplane/controlplane_test.go b/operators/constellation-node-operator/internal/controlplane/controlplane_test.go index a089f02b34..6aba0137bd 100644 --- a/operators/constellation-node-operator/internal/controlplane/controlplane_test.go +++ b/operators/constellation-node-operator/internal/controlplane/controlplane_test.go @@ -11,6 +11,7 @@ import ( "errors" "testing" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -29,7 +30,7 @@ func TestListControlPlaneIPs(t *testing.T) { nodes: []corev1.Node{ { ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""}, + Labels: map[string]string{constants.ControlPlaneRoleLabel: ""}, }, Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{ Type: corev1.NodeInternalIP, diff --git a/operators/constellation-node-operator/internal/etcd/BUILD.bazel b/operators/constellation-node-operator/internal/etcd/BUILD.bazel index f41f5bbf5e..f4e6ced77d 100644 --- a/operators/constellation-node-operator/internal/etcd/BUILD.bazel +++ b/operators/constellation-node-operator/internal/etcd/BUILD.bazel @@ -19,6 +19,7 @@ go_test( srcs = ["etcd_test.go"], embed = [":etcd"], deps = [ + "//operators/constellation-node-operator/internal/constants", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@io_etcd_go_etcd_api_v3//etcdserverpb", diff --git a/operators/constellation-node-operator/internal/etcd/etcd_test.go b/operators/constellation-node-operator/internal/etcd/etcd_test.go index 5775140cbc..30e04d817e 100644 --- a/operators/constellation-node-operator/internal/etcd/etcd_test.go +++ b/operators/constellation-node-operator/internal/etcd/etcd_test.go @@ -11,6 +11,7 @@ import ( "errors" "testing" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" @@ -124,7 +125,7 @@ func TestGetInitialEndpoints(t *testing.T) { nodes: []corev1.Node{ { ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""}, + Labels: map[string]string{constants.ControlPlaneRoleLabel: ""}, }, Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{ Type: corev1.NodeInternalIP, diff --git a/operators/constellation-node-operator/internal/node/BUILD.bazel b/operators/constellation-node-operator/internal/node/BUILD.bazel index 3726946ac1..080257817c 100644 --- a/operators/constellation-node-operator/internal/node/BUILD.bazel +++ b/operators/constellation-node-operator/internal/node/BUILD.bazel @@ -8,6 +8,7 @@ go_library( visibility = ["//operators/constellation-node-operator:__subpackages__"], deps = [ "//operators/constellation-node-operator/api/v1alpha1", + "//operators/constellation-node-operator/internal/constants", "@io_k8s_api//core/v1:core", ], ) @@ -18,6 +19,7 @@ go_test( embed = [":node"], deps = [ "//operators/constellation-node-operator/api/v1alpha1", + "//operators/constellation-node-operator/internal/constants", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@io_k8s_api//core/v1:core", diff --git a/operators/constellation-node-operator/internal/node/node.go b/operators/constellation-node-operator/internal/node/node.go index 854b49906f..ecfcb61587 100644 --- a/operators/constellation-node-operator/internal/node/node.go +++ b/operators/constellation-node-operator/internal/node/node.go @@ -11,11 +11,10 @@ import ( "regexp" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" corev1 "k8s.io/api/core/v1" ) -const controlPlaneRoleLabel = "node-role.kubernetes.io/control-plane" - var reservedHostRegex = regexp.MustCompile(`^(.+\.|)(kubernetes|k8s)\.io(/.*)?$`) // Ready checks if a kubernetes node has the `NodeReady` condition set to true. @@ -40,7 +39,7 @@ func VPCIP(node *corev1.Node) (string, error) { // IsControlPlaneNode returns true if the node is a control plane node. func IsControlPlaneNode(node *corev1.Node) bool { - _, ok := node.Labels[controlPlaneRoleLabel] + _, ok := node.Labels[constants.ControlPlaneRoleLabel] return ok } diff --git a/operators/constellation-node-operator/internal/node/node_test.go b/operators/constellation-node-operator/internal/node/node_test.go index 8a11a19aa8..34b706b5e3 100644 --- a/operators/constellation-node-operator/internal/node/node_test.go +++ b/operators/constellation-node-operator/internal/node/node_test.go @@ -10,6 +10,7 @@ import ( "testing" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -69,7 +70,7 @@ func TestIsControlPlaneNode(t *testing.T) { "node with control-plane role label": { node: corev1.Node{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{controlPlaneRoleLabel: ""}, + Labels: map[string]string{constants.ControlPlaneRoleLabel: ""}, }, }, wantControlPlane: true, From bf7fc6fb1f43b7be43501bf86e2825de5f4bed50 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Sun, 20 Oct 2024 22:19:30 +0200 Subject: [PATCH 13/15] operator: stub etcd remove calls in env tests --- .../controllers/nodeversion_controller_test.go | 8 ++++++++ .../constellation-node-operator/controllers/suite_test.go | 3 +++ 2 files changed, 11 insertions(+) diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go index e32b99af3d..383e8593b5 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go @@ -891,3 +891,11 @@ func (*unimplementedNodeReplacer) CreateNode(_ context.Context, _ string) (nodeN func (*unimplementedNodeReplacer) DeleteNode(_ context.Context, _ string) error { panic("unimplemented") } + +type stubEtcdRemover struct { + deleteErr error +} + +func (r *stubEtcdRemover) RemoveEtcdMemberFromCluster(_ context.Context, _ string) error { + return r.deleteErr +} diff --git a/operators/constellation-node-operator/controllers/suite_test.go b/operators/constellation-node-operator/controllers/suite_test.go index dd2df34078..7b4e232181 100644 --- a/operators/constellation-node-operator/controllers/suite_test.go +++ b/operators/constellation-node-operator/controllers/suite_test.go @@ -123,6 +123,7 @@ var _ = BeforeSuite(func() { err = (&NodeVersionReconciler{ kubernetesServerVersionGetter: fakes.k8sVerGetter, nodeReplacer: fakes.nodeReplacer, + etcdRemover: fakes.etcdRemover, Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), }).SetupWithManager(k8sManager) @@ -147,6 +148,7 @@ type fakeCollection struct { nodeStateGetter *stubNodeStateGetter nodeReplacer *stubNodeReplacer k8sVerGetter *stubKubernetesServerVersionGetter + etcdRemover *stubEtcdRemover clock *testclock.FakeClock } @@ -162,6 +164,7 @@ func newFakes() fakeCollection { nodeStateGetter: &stubNodeStateGetter{}, nodeReplacer: &stubNodeReplacer{}, k8sVerGetter: &stubKubernetesServerVersionGetter{}, + etcdRemover: &stubEtcdRemover{}, clock: testclock.NewFakeClock(time.Now()), } } From eecd9718874ed2171d2ff06e3093c63ced020ffd Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Sun, 20 Oct 2024 22:28:06 +0200 Subject: [PATCH 14/15] operator: upgrade control plane nodes first Before we call out ot the cloud provider we check if there are still control plane nodes that are outdated (or donors). If there are, we don't create any worker nodes, even if we have the budget to do so. --- .../controllers/nodeversion_controller.go | 67 ++++++++++++++++--- .../nodeversion_controller_test.go | 50 +++++++++++++- 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller.go b/operators/constellation-node-operator/controllers/nodeversion_controller.go index cfa3e315e2..a9508e5711 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller.go @@ -136,9 +136,9 @@ func (r *NodeVersionReconciler) Reconcile(ctx context.Context, req ctrl.Request) logr.Error(err, "Unable to list scaling groups") return ctrl.Result{}, err } - scalingGroupByID := make(map[string]updatev1alpha1.ScalingGroup, len(scalingGroupList.Items)) + scalingGroupByID := make(scalingGroupByID, len(scalingGroupList.Items)) for _, scalingGroup := range scalingGroupList.Items { - scalingGroupByID[strings.ToLower(scalingGroup.Spec.GroupID)] = scalingGroup + scalingGroupByID.put(scalingGroup) } annotatedNodes, invalidNodes := r.annotateNodes(ctx, nodeList.Items) groups := groupNodes(annotatedNodes, pendingNodeList.Items, desiredNodeVersion.Spec.ImageReference, desiredNodeVersion.Spec.KubernetesComponentsReference) @@ -217,7 +217,7 @@ func (r *NodeVersionReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{Requeue: shouldRequeue}, nil } - newNodeConfig := newNodeConfig{desiredNodeVersion, groups.Outdated, pendingNodeList.Items, scalingGroupByID, newNodesBudget} + newNodeConfig := newNodeConfig{desiredNodeVersion, groups, pendingNodeList.Items, scalingGroupByID, newNodesBudget} if err := r.createNewNodes(ctx, newNodeConfig); err != nil { logr.Error(err, "Creating new nodes") return ctrl.Result{Requeue: shouldRequeue}, nil @@ -613,12 +613,21 @@ func (r *NodeVersionReconciler) deleteNode(ctx context.Context, controller metav // createNewNodes creates new nodes using up to date images as replacement for outdated nodes. func (r *NodeVersionReconciler) createNewNodes(ctx context.Context, config newNodeConfig) error { + hasOutdatedControlPlanes := func() bool { + for _, entry := range append(config.groups.Outdated, config.groups.Donors...) { + if nodeutil.IsControlPlaneNode(&entry) { + return true + } + } + return false + }() + logr := log.FromContext(ctx) - if config.newNodesBudget < 1 || len(config.outdatedNodes) == 0 { + if config.newNodesBudget < 1 || len(config.groups.Outdated) == 0 { return nil } outdatedNodesPerScalingGroup := make(map[string]int) - for _, node := range config.outdatedNodes { + for _, node := range config.groups.Outdated { // skip outdated nodes that got assigned an heir in this Reconcile call if len(node.Annotations[heirAnnotation]) != 0 { continue @@ -640,17 +649,22 @@ func (r *NodeVersionReconciler) createNewNodes(ctx context.Context, config newNo requiredNodesPerScalingGroup[scalingGroupID] = outdatedNodesPerScalingGroup[scalingGroupID] - pendingJoiningNodesPerScalingGroup[scalingGroupID] } } - for scalingGroupID := range requiredNodesPerScalingGroup { - scalingGroup, ok := config.scalingGroupByID[scalingGroupID] + for _, scalingGroupID := range config.scalingGroupByID.getScalingGroupIDsSorted() { + scalingGroup, ok := config.scalingGroupByID.get(scalingGroupID) if !ok { logr.Info("Scaling group does not have matching resource", "scalingGroup", scalingGroupID, "scalingGroups", config.scalingGroupByID) continue } + if requiredNodesPerScalingGroup[scalingGroupID] == 0 { + logr.Info("No new nodes needed for scaling group", "scalingGroup", scalingGroupID) + continue + } if !strings.EqualFold(scalingGroup.Status.ImageReference, config.desiredNodeVersion.Spec.ImageReference) { logr.Info("Scaling group does not use latest image", "scalingGroup", scalingGroupID, "usedImage", scalingGroup.Status.ImageReference, "wantedImage", config.desiredNodeVersion.Spec.ImageReference) continue } - if requiredNodesPerScalingGroup[scalingGroupID] == 0 { + if hasOutdatedControlPlanes && scalingGroup.Spec.Role != updatev1alpha1.ControlPlaneRole { + logr.Info("There are still outdated control plane nodes which must be replaced first before this worker scaling group is upgraded", "scalingGroup", scalingGroupID) continue } for { @@ -679,10 +693,16 @@ func (r *NodeVersionReconciler) createNewNodes(ctx context.Context, config newNo if err := ctrl.SetControllerReference(&config.desiredNodeVersion, pendingNode, r.Scheme); err != nil { return err } + // Note that while we call Create here, it is not certain that the next reconciler iteration + // will see the pending node. This is because of delays in the kubeAPI. + // Currently, we don't explicitly wait until we can Get the resource. + // We can never be certain that other calls to Get will also see the resource, + // therefore it's just a tradeoff of the probability that we create more nodes than + // the specified budget. if err := r.Create(ctx, pendingNode); err != nil { return err } - logr.Info("Created new node", "createdNode", nodeName, "scalingGroup", scalingGroupID) + logr.Info("Created new node", "createdNode", nodeName, "scalingGroup", scalingGroupID, "requiredNodes", requiredNodesPerScalingGroup[scalingGroupID]) requiredNodesPerScalingGroup[scalingGroupID]-- config.newNodesBudget-- } @@ -939,10 +959,35 @@ type kubernetesServerVersionGetter interface { ServerVersion() (*version.Info, error) } +type scalingGroupByID map[string]updatev1alpha1.ScalingGroup + +// getScalingGroupIDsSorted returns the group IDs of all scaling groups with +// scaling groups with the role "control-plane" first. +func (s scalingGroupByID) getScalingGroupIDsSorted() []string { + var controlPlaneGroupIDs, otherGroupIDs []string + for id := range s { + if s[id].Spec.Role == updatev1alpha1.ControlPlaneRole { + controlPlaneGroupIDs = append(controlPlaneGroupIDs, id) + } else { + otherGroupIDs = append(otherGroupIDs, id) + } + } + return append(controlPlaneGroupIDs, otherGroupIDs...) +} + +func (s scalingGroupByID) put(scalingGroup updatev1alpha1.ScalingGroup) { + s[strings.ToLower(scalingGroup.Spec.GroupID)] = scalingGroup +} + +func (s scalingGroupByID) get(scalingGroupID string) (scalingGroup updatev1alpha1.ScalingGroup, ok bool) { + scalingGroup, ok = s[strings.ToLower(scalingGroupID)] + return +} + type newNodeConfig struct { desiredNodeVersion updatev1alpha1.NodeVersion - outdatedNodes []corev1.Node + groups nodeGroups pendingNodes []updatev1alpha1.PendingNode - scalingGroupByID map[string]updatev1alpha1.ScalingGroup + scalingGroupByID scalingGroupByID newNodesBudget uint32 } diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go index 383e8593b5..b30bbc4191 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go @@ -560,6 +560,51 @@ func TestCreateNewNodes(t *testing.T) { budget: 2, wantCreateCalls: []string{"scaling-group"}, }, + "outdated nodes still contain control plane nodes": { + outdatedNodes: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "control-plane-node", + Annotations: map[string]string{ + scalingGroupAnnotation: "control-plane-scaling-group", + }, + Labels: map[string]string{ + constants.ControlPlaneRoleLabel: "", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + Annotations: map[string]string{ + scalingGroupAnnotation: "scaling-group", + }, + }, + }, + }, + scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ + "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + Role: updatev1alpha1.WorkerRole, + }, + Status: updatev1alpha1.ScalingGroupStatus{ + ImageReference: "image", + }, + }, + "control-plane-scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "control-plane-scaling-group", + Role: updatev1alpha1.ControlPlaneRole, + }, + Status: updatev1alpha1.ScalingGroupStatus{ + ImageReference: "image", + }, + }, + }, + budget: 2, + wantCreateCalls: []string{"control-plane-scaling-group"}, + }, "scaling group does not exist": { outdatedNodes: []corev1.Node{ { @@ -592,7 +637,10 @@ func TestCreateNewNodes(t *testing.T) { }, Scheme: getScheme(t), } - newNodeConfig := newNodeConfig{desiredNodeImage, tc.outdatedNodes, tc.pendingNodes, tc.scalingGroupByID, tc.budget} + groups := nodeGroups{ + Outdated: tc.outdatedNodes, + } + newNodeConfig := newNodeConfig{desiredNodeImage, groups, tc.pendingNodes, tc.scalingGroupByID, tc.budget} err := reconciler.createNewNodes(context.Background(), newNodeConfig) require.NoError(err) assert.Equal(tc.wantCreateCalls, reconciler.nodeReplacer.(*stubNodeReplacerWriter).createCalls) From 72f85f392fe8c748b5fc7ee97c6738c0b209a5f3 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Sun, 20 Oct 2024 22:31:34 +0200 Subject: [PATCH 15/15] operator: add test that tries to upgrade worker nodes first and fails --- .../nodeversion_controller_env_test.go | 494 +++++++++++------- .../nodeversion_controller_test.go | 56 +- .../pendingnode_controller_env_test.go | 14 +- .../pendingnode_controller_test.go | 26 +- .../controllers/suite_test.go | 2 +- 5 files changed, 379 insertions(+), 213 deletions(-) diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller_env_test.go b/operators/constellation-node-operator/controllers/nodeversion_controller_env_test.go index 7fa62ce3a1..1e1353bc67 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller_env_test.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller_env_test.go @@ -27,6 +27,7 @@ import ( nodemaintenancev1beta1 "github.com/edgelesssys/constellation/v2/3rdparty/node-maintenance-operator/api/v1beta1" mainconstants "github.com/edgelesssys/constellation/v2/internal/constants" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" ) var _ = Describe("NodeVersion controller", func() { @@ -38,10 +39,15 @@ var _ = Describe("NodeVersion controller", func() { // Define utility constants for object names and testing timeouts/durations and intervals. const ( - nodeVersionResourceName = "nodeversion" - firstNodeName = "node-1" - secondNodeName = "node-2" - scalingGroupID = "scaling-group" + nodeVersionResourceName = "nodeversion" + firstWorkerNodeName = "worker-node-1" + secondWorkerNodeName = "worker-node-2" + thirdWorkerNodeName = "worker-node-3" + firstControlPlaneNodeName = "control-plane-node-1" + secondControlPlaneNodeName = "control-plane-node-2" + thirdControlPlaneNodeName = "control-plane-node-3" + scalingGroupIDWorker = "scaling-group-worker" + scalingGroupIDControlPlane = "scaling-group-control-plane" timeout = time.Second * 20 duration = time.Second * 2 @@ -52,17 +58,17 @@ var _ = Describe("NodeVersion controller", func() { KubernetesComponentsReference: "ref-1", } - firstNodeLookupKey := types.NamespacedName{Name: firstNodeName} - secondNodeLookupKey := types.NamespacedName{Name: secondNodeName} + firstNodeLookupKeyWorker := types.NamespacedName{Name: firstWorkerNodeName} + secondNodeLookupKeyWorker := types.NamespacedName{Name: secondWorkerNodeName} + firstNodeLookupKeyControlPlane := types.NamespacedName{Name: firstControlPlaneNodeName} + secondNodeLookupKeyControlPlane := types.NamespacedName{Name: secondControlPlaneNodeName} nodeVersionLookupKey := types.NamespacedName{Name: nodeVersionResourceName} - scalingGroupLookupKey := types.NamespacedName{Name: scalingGroupID} - joiningPendingNodeLookupKey := types.NamespacedName{Name: secondNodeName} - nodeMaintenanceLookupKey := types.NamespacedName{Name: firstNodeName} + scalingGroupLookupKeyWorker := types.NamespacedName{Name: scalingGroupIDWorker} + scalingGroupLookupKeyControlPlane := types.NamespacedName{Name: scalingGroupIDControlPlane} Context("When updating the cluster-wide node version", func() { - testNodeVersionUpdate := func(newNodeVersionSpec updatev1alpha1.NodeVersionSpec) { + testNodeVersionUpdate := func(newNodeVersionSpec updatev1alpha1.NodeVersionSpec, workersFirst bool) { By("creating a node version resource specifying the first node version") - Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupID, firstVersionSpec.ImageReference)).Should(Succeed()) nodeVersion := &updatev1alpha1.NodeVersion{ TypeMeta: metav1.TypeMeta{ APIVersion: "update.edgeless.systems/v1alpha1", @@ -75,16 +81,48 @@ var _ = Describe("NodeVersion controller", func() { } Expect(k8sClient.Create(ctx, nodeVersion)).Should(Succeed()) - By("creating a node resource using the first node image") - fakes.nodeReplacer.setNodeImage(firstNodeName, firstVersionSpec.ImageReference) - fakes.nodeReplacer.setScalingGroupID(firstNodeName, scalingGroupID) - firstNode := &corev1.Node{ + By("creating a control plane node resource using the first node image") + fakes.nodeReplacer.setNodeImage(firstControlPlaneNodeName, firstVersionSpec.ImageReference) + fakes.nodeReplacer.setScalingGroupID(firstControlPlaneNodeName, scalingGroupIDControlPlane) + firstControlPlaneNode := &corev1.Node{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Node", }, ObjectMeta: metav1.ObjectMeta{ - Name: firstNodeName, + Name: firstControlPlaneNodeName, + Labels: map[string]string{ + "custom-node-label": "custom-node-label-value", + constants.ControlPlaneRoleLabel: "", + }, + Annotations: map[string]string{ + mainconstants.NodeKubernetesComponentsAnnotationKey: firstVersionSpec.KubernetesComponentsReference, + }, + }, + Spec: corev1.NodeSpec{ + ProviderID: firstControlPlaneNodeName, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: "192.0.2.1", + }, + }, + }, + } + Expect(k8sClient.Create(ctx, firstControlPlaneNode)).Should(Succeed()) + + By("creating a worker node resource using the first node image") + fakes.nodeReplacer.setNodeImage(firstWorkerNodeName, firstVersionSpec.ImageReference) + fakes.nodeReplacer.setScalingGroupID(firstWorkerNodeName, scalingGroupIDWorker) + firstWorkerNode := &corev1.Node{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: firstWorkerNodeName, Labels: map[string]string{ "custom-node-label": "custom-node-label-value", }, @@ -93,24 +131,40 @@ var _ = Describe("NodeVersion controller", func() { }, }, Spec: corev1.NodeSpec{ - ProviderID: firstNodeName, + ProviderID: firstWorkerNodeName, }, } - Expect(k8sClient.Create(ctx, firstNode)).Should(Succeed()) + Expect(k8sClient.Create(ctx, firstWorkerNode)).Should(Succeed()) - By("creating a scaling group resource using the first node image") - Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupID, firstVersionSpec.ImageReference)).Should(Succeed()) - scalingGroup := &updatev1alpha1.ScalingGroup{ + By("creating worker scaling group resource using the first node image") + Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupIDWorker, firstVersionSpec.ImageReference)).Should(Succeed()) + scalingGroupWorker := &updatev1alpha1.ScalingGroup{ ObjectMeta: metav1.ObjectMeta{ - Name: scalingGroupID, + Name: scalingGroupIDWorker, }, Spec: updatev1alpha1.ScalingGroupSpec{ NodeVersion: nodeVersionResourceName, - GroupID: scalingGroupID, + GroupID: scalingGroupIDWorker, Autoscaling: true, + Role: updatev1alpha1.WorkerRole, }, } - Expect(k8sClient.Create(ctx, scalingGroup)).Should(Succeed()) + Expect(k8sClient.Create(ctx, scalingGroupWorker)).Should(Succeed()) + + By("creating control plane scaling group resource using the first node image") + Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupIDControlPlane, firstVersionSpec.ImageReference)).Should(Succeed()) + scalingGroupControlPlane := &updatev1alpha1.ScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: scalingGroupIDControlPlane, + }, + Spec: updatev1alpha1.ScalingGroupSpec{ + NodeVersion: nodeVersionResourceName, + GroupID: scalingGroupIDControlPlane, + Autoscaling: true, + Role: updatev1alpha1.ControlPlaneRole, + }, + } + Expect(k8sClient.Create(ctx, scalingGroupControlPlane)).Should(Succeed()) By("creating a cluster-autoscaler deployment") ctx := context.Background() @@ -167,11 +221,20 @@ var _ = Describe("NodeVersion controller", func() { return 0 } return len(nodeVersion.Status.UpToDate) - }, timeout, interval).Should(Equal(1)) + }, timeout, interval).Should(Equal(2)) By("updating the node version") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateReady) - fakes.nodeReplacer.setCreatedNode(secondNodeName, secondNodeName, nil) + fakes.nodeReplacer.addCreatedNode(scalingGroupIDControlPlane, secondControlPlaneNodeName, secondControlPlaneNodeName, nil) + fakes.nodeReplacer.addCreatedNode(scalingGroupIDWorker, secondWorkerNodeName, secondWorkerNodeName, nil) + fakes.nodeStateGetter.setNodeState(secondControlPlaneNodeName, updatev1alpha1.NodeStateReady) + fakes.nodeStateGetter.setNodeState(secondWorkerNodeName, updatev1alpha1.NodeStateReady) + // When the pending node CR that the nodeversion loop creates is not detected on the second iteration e.g., because of delay in the KubeAPI + // it creates a second node via the cloud provider api. This is fine because the pending node/mint node is not matched if it's not needed + // and the nodeversion loop will clean up the mint node. + fakes.nodeStateGetter.setNodeState(thirdControlPlaneNodeName, updatev1alpha1.NodeStateCreating) + fakes.nodeStateGetter.setNodeState(thirdWorkerNodeName, updatev1alpha1.NodeStateCreating) + fakes.nodeReplacer.addCreatedNode(scalingGroupIDControlPlane, thirdControlPlaneNodeName, thirdControlPlaneNodeName, nil) + fakes.nodeReplacer.addCreatedNode(scalingGroupIDWorker, thirdWorkerNodeName, thirdWorkerNodeName, nil) // Eventually the node version with the new NodeVersion spec. Eventually(func() error { if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { @@ -181,172 +244,216 @@ var _ = Describe("NodeVersion controller", func() { return k8sClient.Update(ctx, nodeVersion) }, timeout, interval).Should(Succeed()) - By("checking that there is an outdated node in the status") + By("checking that there are 2 outdated node in the status") Eventually(func() int { if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { return 0 } return len(nodeVersion.Status.Outdated) - }, timeout, interval).Should(Equal(1), "outdated nodes should be 1") + }, timeout, interval).Should(Equal(2), "outdated nodes should be 2") - By("checking that the scaling group is up to date") + By("checking that the control plane scaling group is up to date") Eventually(func() string { - if err := k8sClient.Get(ctx, scalingGroupLookupKey, scalingGroup); err != nil { + if err := k8sClient.Get(ctx, scalingGroupLookupKeyControlPlane, scalingGroupControlPlane); err != nil { return "" } - return scalingGroup.Status.ImageReference + return scalingGroupControlPlane.Status.ImageReference }, timeout, interval).Should(Equal(newNodeVersionSpec.ImageReference)) - By("checking that a pending node is created") - pendingNode := &updatev1alpha1.PendingNode{} - Eventually(func() error { - return k8sClient.Get(ctx, joiningPendingNodeLookupKey, pendingNode) - }, timeout, interval).Should(Succeed()) - Eventually(func() updatev1alpha1.CSPNodeState { - _ = k8sClient.Get(ctx, joiningPendingNodeLookupKey, pendingNode) - return pendingNode.Status.CSPNodeState - }, timeout, interval).Should(Equal(updatev1alpha1.NodeStateReady)) - Eventually(func() int { - if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { - return 0 - } - return len(nodeVersion.Status.Pending) - }, timeout, interval).Should(Equal(1)) - - By("creating a new node resource using the image from the new node version") - fakes.nodeReplacer.setNodeImage(secondNodeName, newNodeVersionSpec.ImageReference) - fakes.nodeReplacer.setScalingGroupID(secondNodeName, scalingGroupID) - secondNode := &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: secondNodeName, - }, - Spec: corev1.NodeSpec{ - ProviderID: secondNodeName, - }, - } - Expect(k8sClient.Create(ctx, secondNode)).Should(Succeed()) - - By("marking the new node as AwaitingAnnotation") - Eventually(func() int { - err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion) - if err != nil { - return 0 - } - return len(nodeVersion.Status.AwaitingAnnotation) - }, timeout, interval).Should(Equal(1)) - // add a JoiningNode CR for the new node - joiningNode := &updatev1alpha1.JoiningNode{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "update.edgeless.systems/v1alpha1", - Kind: "JoiningNode", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: secondNodeName, - }, - Spec: updatev1alpha1.JoiningNodeSpec{ - Name: secondNodeName, - ComponentsReference: newNodeVersionSpec.KubernetesComponentsReference, - }, - } - Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed()) - Eventually(func() int { - err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion) - if err != nil { - return 1 - } - return len(nodeVersion.Status.AwaitingAnnotation) - }, timeout, interval).Should(Equal(0)) - - By("checking that the new node is properly annotated") - Eventually(func() error { - if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { - return err - } - // check nodeImageAnnotation annotation - if _, ok := secondNode.Annotations[nodeImageAnnotation]; !ok { - return fmt.Errorf("node %s is missing %s annotation", secondNode.Name, nodeImageAnnotation) - } - // check mainconstants.NodeKubernetesComponentsAnnotationKey annotation - if _, ok := secondNode.Annotations[mainconstants.NodeKubernetesComponentsAnnotationKey]; !ok { - return fmt.Errorf("node %s is missing %s annotation", secondNode.Name, mainconstants.NodeKubernetesComponentsAnnotationKey) - } - return nil - }, timeout, interval).Should(Succeed()) - - By("checking that the nodes are paired as donor and heir") - Eventually(func() map[string]string { - if err := k8sClient.Get(ctx, firstNodeLookupKey, firstNode); err != nil { - return nil - } - return firstNode.Annotations - }, timeout, interval).Should(HaveKeyWithValue(heirAnnotation, secondNodeName)) - Eventually(func() map[string]string { - if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { - return nil - } - return secondNode.Annotations - }, timeout, interval).Should(HaveKeyWithValue(donorAnnotation, firstNodeName)) - - Eventually(func() error { - if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { - return err - } - if len(nodeVersion.Status.Donors) != 1 { - return fmt.Errorf("node version %s has %d donors, expected 1", nodeVersion.Name, len(nodeVersion.Status.Donors)) - } - if len(nodeVersion.Status.Heirs) != 1 { - return fmt.Errorf("node version %s has %d heirs, expected 1", nodeVersion.Name, len(nodeVersion.Status.Heirs)) + By("checking that the worker scaling group is up to date") + Eventually(func() string { + if err := k8sClient.Get(ctx, scalingGroupLookupKeyWorker, scalingGroupWorker); err != nil { + return "" } - return nil - }, timeout, interval).Should(Succeed()) - Expect(k8sClient.Get(ctx, joiningPendingNodeLookupKey, pendingNode)).Should(Not(Succeed())) + return scalingGroupWorker.Status.ImageReference + }, timeout, interval).Should(Equal(newNodeVersionSpec.ImageReference)) - By("checking that node labels are copied to the heir") - Eventually(func() map[string]string { - if err := k8sClient.Get(ctx, firstNodeLookupKey, firstNode); err != nil { + replaceNodeTest := func(firstNodeLookupKey, secondNodeLookupKey types.NamespacedName, firstNodeName, secondNodeName, scalingGroupID string, firstNode *corev1.Node, expectPendingNode bool) *corev1.Node { + By("checking that the pending node is created") + pendingNode := &updatev1alpha1.PendingNode{} + // If we try to upgrade the worker nodes first, we expect here that the worker node is not created + if !expectPendingNode { + Consistently(func() bool { + if err := k8sClient.Get(ctx, secondNodeLookupKey, pendingNode); err != nil { + return false + } + return true + }, duration, interval).Should(BeFalse()) return nil } - return firstNode.Labels - }, timeout, interval).Should(HaveKeyWithValue("custom-node-label", "custom-node-label-value")) - - By("marking the new node as ready") - Eventually(func() error { - if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { - return err + Eventually(func() bool { + if err := k8sClient.Get(ctx, secondNodeLookupKey, pendingNode); err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + Eventually(func() updatev1alpha1.CSPNodeState { + _ = k8sClient.Get(ctx, secondNodeLookupKey, pendingNode) + return pendingNode.Status.CSPNodeState + }, timeout, interval).Should(Equal(updatev1alpha1.NodeStateReady)) + Eventually(func() bool { + if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { + return false + } + return len(nodeVersion.Status.Pending) >= 1 + }, timeout, interval).Should(BeTrue()) + + By("creating a new node resource using the image from the new node version") + fakes.nodeReplacer.setNodeImage(secondNodeName, newNodeVersionSpec.ImageReference) + fakes.nodeReplacer.setScalingGroupID(secondNodeName, scalingGroupID) + var labels map[string]string + if _, ok := firstNode.Labels[constants.ControlPlaneRoleLabel]; ok { + labels = map[string]string{ + constants.ControlPlaneRoleLabel: "", + } } - secondNode.Status.Conditions = []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionTrue, + secondNode := &corev1.Node{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secondNodeName, + Labels: labels, + }, + Spec: corev1.NodeSpec{ + ProviderID: secondNodeName, }, } - return k8sClient.Status().Update(ctx, secondNode) - }, timeout, interval).Should(Succeed()) - - By("waiting for a NodeMaintenance resource to be created") - nodeMaintenance := &nodemaintenancev1beta1.NodeMaintenance{} - Eventually(func() error { - return k8sClient.Get(ctx, nodeMaintenanceLookupKey, nodeMaintenance) - }, timeout, interval).Should(Succeed()) - - By("marking the NodeMaintenance as successful") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateTerminated) - Eventually(func() error { - if err := k8sClient.Get(ctx, nodeMaintenanceLookupKey, nodeMaintenance); err != nil { - return err + Expect(k8sClient.Create(ctx, secondNode)).Should(Succeed()) + + By("marking the new node as AwaitingAnnotation") + Eventually(func() int { + err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion) + if err != nil { + return 0 + } + return len(nodeVersion.Status.AwaitingAnnotation) + }, timeout, interval).Should(Equal(1)) + // add a JoiningNode CR for the new node + joiningNode := &updatev1alpha1.JoiningNode{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "update.edgeless.systems/v1alpha1", + Kind: "JoiningNode", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secondNodeName, + }, + Spec: updatev1alpha1.JoiningNodeSpec{ + Name: secondNodeName, + ComponentsReference: newNodeVersionSpec.KubernetesComponentsReference, + }, } - nodeMaintenance.Status.Phase = nodemaintenancev1beta1.MaintenanceSucceeded - return k8sClient.Status().Update(ctx, nodeMaintenance) - }, timeout, interval).Should(Succeed()) + Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed()) + Eventually(func() int { + err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion) + if err != nil { + return 1 + } + return len(nodeVersion.Status.AwaitingAnnotation) + }, timeout, interval).Should(Equal(0)) + + By("checking that the new node is properly annotated") + Eventually(func() error { + if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { + return err + } + // check nodeImageAnnotation annotation + if _, ok := secondNode.Annotations[nodeImageAnnotation]; !ok { + return fmt.Errorf("node %s is missing %s annotation", secondNode.Name, nodeImageAnnotation) + } + // check mainconstants.NodeKubernetesComponentsAnnotationKey annotation + if _, ok := secondNode.Annotations[mainconstants.NodeKubernetesComponentsAnnotationKey]; !ok { + return fmt.Errorf("node %s is missing %s annotation", secondNode.Name, mainconstants.NodeKubernetesComponentsAnnotationKey) + } + return nil + }, timeout, interval).Should(Succeed()) + + By("checking that the nodes are paired as donor and heir") + Eventually(func() map[string]string { + if err := k8sClient.Get(ctx, firstNodeLookupKey, firstNode); err != nil { + return nil + } + return firstNode.Annotations + }, timeout, interval).Should(HaveKeyWithValue(heirAnnotation, secondNodeName)) + Eventually(func() map[string]string { + if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { + return nil + } + return secondNode.Annotations + }, timeout, interval).Should(HaveKeyWithValue(donorAnnotation, firstNodeName)) + + Eventually(func() error { + if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { + return err + } + if len(nodeVersion.Status.Donors) != 1 { + return fmt.Errorf("node version %s has %d donors, expected 1", nodeVersion.Name, len(nodeVersion.Status.Donors)) + } + if len(nodeVersion.Status.Heirs) != 1 { + return fmt.Errorf("node version %s has %d heirs, expected 1", nodeVersion.Name, len(nodeVersion.Status.Heirs)) + } + return nil + }, timeout, interval).Should(Succeed()) + Expect(k8sClient.Get(ctx, secondNodeLookupKey, pendingNode)).Should(Not(Succeed())) + + By("checking that node labels are copied to the heir") + Eventually(func() map[string]string { + if err := k8sClient.Get(ctx, firstNodeLookupKey, firstNode); err != nil { + return nil + } + return firstNode.Labels + }, timeout, interval).Should(HaveKeyWithValue("custom-node-label", "custom-node-label-value")) + + By("marking the new node as ready") + Eventually(func() error { + if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { + return err + } + secondNode.Status.Conditions = []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + } + return k8sClient.Status().Update(ctx, secondNode) + }, timeout, interval).Should(Succeed()) + + By("waiting for a NodeMaintenance resource to be created") + nodeMaintenance := &nodemaintenancev1beta1.NodeMaintenance{} + Eventually(func() error { + return k8sClient.Get(ctx, firstNodeLookupKey, nodeMaintenance) + }, timeout, interval).Should(Succeed()) + + By("marking the NodeMaintenance as successful") + fakes.nodeStateGetter.setNodeState(firstNodeName, updatev1alpha1.NodeStateTerminated) + Eventually(func() error { + if err := k8sClient.Get(ctx, firstNodeLookupKey, nodeMaintenance); err != nil { + return err + } + nodeMaintenance.Status.Phase = nodemaintenancev1beta1.MaintenanceSucceeded + return k8sClient.Status().Update(ctx, nodeMaintenance) + }, timeout, interval).Should(Succeed()) + + By("checking that the outdated node is removed") + Eventually(func() error { + return k8sClient.Get(ctx, firstNodeLookupKey, firstNode) + }, timeout, interval).Should(Not(Succeed())) + + return secondNode + } - By("checking that the outdated node is removed") - Eventually(func() error { - return k8sClient.Get(ctx, firstNodeLookupKey, firstNode) - }, timeout, interval).Should(Not(Succeed())) + var createdControlPlane *corev1.Node + var createdWorkerNode *corev1.Node + if workersFirst { + _ = replaceNodeTest(firstNodeLookupKeyWorker, secondNodeLookupKeyWorker, firstWorkerNodeName, secondWorkerNodeName, scalingGroupIDWorker, firstWorkerNode, false) + createdControlPlane = replaceNodeTest(firstNodeLookupKeyControlPlane, secondNodeLookupKeyControlPlane, firstControlPlaneNodeName, secondControlPlaneNodeName, scalingGroupIDControlPlane, firstControlPlaneNode, true) + createdWorkerNode = replaceNodeTest(firstNodeLookupKeyWorker, secondNodeLookupKeyWorker, firstWorkerNodeName, secondWorkerNodeName, scalingGroupIDWorker, firstWorkerNode, true) + } else { + createdControlPlane = replaceNodeTest(firstNodeLookupKeyControlPlane, secondNodeLookupKeyControlPlane, firstControlPlaneNodeName, secondControlPlaneNodeName, scalingGroupIDControlPlane, firstControlPlaneNode, true) + createdWorkerNode = replaceNodeTest(firstNodeLookupKeyWorker, secondNodeLookupKeyWorker, firstWorkerNodeName, secondWorkerNodeName, scalingGroupIDWorker, firstWorkerNode, true) + } By("checking that all nodes are up-to-date") Eventually(func() int { @@ -355,29 +462,48 @@ var _ = Describe("NodeVersion controller", func() { return 0 } return len(nodeVersion.Status.UpToDate) - }, timeout, interval).Should(Equal(1)) + }, timeout, interval).Should(Equal(2)) By("cleaning up all resources") Expect(k8sClient.Delete(ctx, nodeVersion)).Should(Succeed()) - Expect(k8sClient.Delete(ctx, scalingGroup)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, scalingGroupWorker)).Should(Succeed()) Expect(k8sClient.Delete(ctx, autoscalerDeployment)).Should(Succeed()) Expect(k8sClient.Delete(ctx, strategy)).Should(Succeed()) - Expect(k8sClient.Delete(ctx, secondNode)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, createdControlPlane)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, createdWorkerNode)).Should(Succeed()) } When("Updating the image reference", func() { It("Should update every node in the cluster", func() { - testNodeVersionUpdate(updatev1alpha1.NodeVersionSpec{ - ImageReference: "version-2", - KubernetesComponentsReference: "ref-1", - }) + testNodeVersionUpdate( + updatev1alpha1.NodeVersionSpec{ + ImageReference: "version-2", + KubernetesComponentsReference: "ref-1", + }, + false, + ) }) }) When("Updating the Kubernetes components reference", func() { It("Should update every node in the cluster", func() { - testNodeVersionUpdate(updatev1alpha1.NodeVersionSpec{ - ImageReference: "version-1", - KubernetesComponentsReference: "ref-2", - }) + testNodeVersionUpdate( + updatev1alpha1.NodeVersionSpec{ + ImageReference: "version-1", + KubernetesComponentsReference: "ref-2", + }, + false, + ) + }) + }) + When("Updating the Kubernetes components reference and wanting to upgrade the worker nodes first", func() { + It("should fail to update the worker nodes before the control plane nodes", func() { + testNodeVersionUpdate( + updatev1alpha1.NodeVersionSpec{ + ImageReference: "version-1", + KubernetesComponentsReference: "ref-2", + MaxNodeBudget: 2, + }, + true, + ) }) }) }) diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go index b30bbc4191..4f926b978a 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go @@ -9,6 +9,7 @@ package controllers import ( "context" "errors" + "strconv" "sync" "testing" @@ -21,6 +22,7 @@ import ( mainconstants "github.com/edgelesssys/constellation/v2/internal/constants" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" ) func TestAnnotateNodes(t *testing.T) { @@ -800,16 +802,22 @@ func TestGroupNodes(t *testing.T) { assert.Equal(wantNodeGroups, groups) } +// stubNode contains all information usually associated with a node freshly +// created by the cloud provider. +type stubNode struct { + name string + providerID string +} + type stubNodeReplacer struct { sync.RWMutex - nodeImages map[string]string - scalingGroups map[string]string - createNodeName string - createProviderID string - nodeImageErr error - scalingGroupIDErr error - createErr error - deleteErr error + nodeImages map[string]string + scalingGroups map[string]string + createNodesByScalingGroupID map[string][]stubNode + nodeImageErr error + scalingGroupIDErr error + createErr error + deleteErr error } func (r *stubNodeReplacer) GetNodeImage(_ context.Context, providerID string) (string, error) { @@ -824,10 +832,25 @@ func (r *stubNodeReplacer) GetScalingGroupID(_ context.Context, providerID strin return r.scalingGroups[providerID], r.scalingGroupIDErr } -func (r *stubNodeReplacer) CreateNode(_ context.Context, _ string) (nodeName, providerID string, err error) { +// CreateNode stubs the cloud provider API call to create a node. +func (r *stubNodeReplacer) CreateNode(_ context.Context, scalingGroupID string) (nodeName, providerID string, err error) { r.RLock() defer r.RUnlock() - return r.createNodeName, r.createProviderID, r.createErr + nodes, ok := r.createNodesByScalingGroupID[scalingGroupID] + if !ok { + panic("unexpected call to CreateNode with scaling group ID " + scalingGroupID + ", existing scaling group IDs: " + strconv.Itoa(len(r.createNodesByScalingGroupID))) + } + if len(nodes) == 0 { + panic("unexpected call to CreateNode with scaling group ID " + scalingGroupID + ", no nodes left") + } + + nodeName = nodes[0].name + providerID = nodes[0].providerID + err = r.createErr + + r.createNodesByScalingGroupID[scalingGroupID] = nodes[1:] + + return } func (r *stubNodeReplacer) DeleteNode(_ context.Context, _ string) error { @@ -856,12 +879,14 @@ func (r *stubNodeReplacer) setScalingGroupID(providerID, scalingGroupID string) r.scalingGroups[providerID] = scalingGroupID } -func (r *stubNodeReplacer) setCreatedNode(nodeName, providerID string, err error) { +func (r *stubNodeReplacer) addCreatedNode(scalingGroupID, nodeName, providerID string, _ error) { r.Lock() defer r.Unlock() - r.createNodeName = nodeName - r.createProviderID = providerID - r.createErr = err + if r.createNodesByScalingGroupID == nil { + r.createNodesByScalingGroupID = make(map[string][]stubNode) + } + r.createNodesByScalingGroupID[scalingGroupID] = append(r.createNodesByScalingGroupID[scalingGroupID], stubNode{nodeName, providerID}) + r.createErr = nil } func (r *stubNodeReplacer) reset() { @@ -869,8 +894,7 @@ func (r *stubNodeReplacer) reset() { defer r.Unlock() r.nodeImages = nil r.scalingGroups = nil - r.createNodeName = "" - r.createProviderID = "" + r.createNodesByScalingGroupID = nil r.createErr = nil r.deleteErr = nil } diff --git a/operators/constellation-node-operator/controllers/pendingnode_controller_env_test.go b/operators/constellation-node-operator/controllers/pendingnode_controller_env_test.go index 05e5de4eee..d9a6416d06 100644 --- a/operators/constellation-node-operator/controllers/pendingnode_controller_env_test.go +++ b/operators/constellation-node-operator/controllers/pendingnode_controller_env_test.go @@ -64,7 +64,7 @@ var _ = Describe("PendingNode controller", func() { Context("When creating pending node with goal join", func() { It("Should terminate the node after failing to join by the deadline", func() { By("setting the CSP node state to creating") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateCreating) + fakes.nodeStateGetter.setNodeState(pendingNodeName, updatev1alpha1.NodeStateCreating) By("creating a pending node resource") ctx := context.Background() @@ -77,7 +77,7 @@ var _ = Describe("PendingNode controller", func() { Name: pendingNodeName, }, Spec: updatev1alpha1.PendingNodeSpec{ - ProviderID: "provider-id", + ProviderID: pendingNodeName, ScalingGroupID: "scaling-group-id", NodeName: "test-node", Goal: updatev1alpha1.NodeGoalJoin, @@ -120,7 +120,7 @@ var _ = Describe("PendingNode controller", func() { }, timeout, interval).Should(Equal(updatev1alpha1.NodeGoalLeave)) By("setting the CSP node state to terminated") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateTerminated) + fakes.nodeStateGetter.setNodeState(pendingNodeName, updatev1alpha1.NodeStateTerminated) // trigger reconciliation before regular check interval to speed up test by changing the spec Eventually(func() error { if err := k8sClient.Get(ctx, pendingNodeLookupKey, pendingNode); err != nil { @@ -138,7 +138,7 @@ var _ = Describe("PendingNode controller", func() { It("Should should detect successful node join", func() { By("setting the CSP node state to creating") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateCreating) + fakes.nodeStateGetter.setNodeState(pendingNodeName, updatev1alpha1.NodeStateCreating) By("creating a pending node resource") ctx := context.Background() @@ -151,7 +151,7 @@ var _ = Describe("PendingNode controller", func() { Name: pendingNodeName, }, Spec: updatev1alpha1.PendingNodeSpec{ - ProviderID: "provider-id", + ProviderID: pendingNodeName, ScalingGroupID: "scaling-group-id", NodeName: "test-node", Goal: updatev1alpha1.NodeGoalJoin, @@ -178,7 +178,7 @@ var _ = Describe("PendingNode controller", func() { }, timeout, interval).Should(Equal(updatev1alpha1.NodeStateCreating)) By("setting the CSP node state to ready") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateReady) + fakes.nodeStateGetter.setNodeState(pendingNodeName, updatev1alpha1.NodeStateReady) By("creating a new node resource with the same node name and provider ID") node := &corev1.Node{ @@ -190,7 +190,7 @@ var _ = Describe("PendingNode controller", func() { Name: "test-node", }, Spec: corev1.NodeSpec{ - ProviderID: "provider-id", + ProviderID: pendingNodeName, }, } Expect(k8sClient.Create(ctx, node)).Should(Succeed()) diff --git a/operators/constellation-node-operator/controllers/pendingnode_controller_test.go b/operators/constellation-node-operator/controllers/pendingnode_controller_test.go index 1a564af768..d5ca55fe6d 100644 --- a/operators/constellation-node-operator/controllers/pendingnode_controller_test.go +++ b/operators/constellation-node-operator/controllers/pendingnode_controller_test.go @@ -231,15 +231,20 @@ func TestReachedGoal(t *testing.T) { type stubNodeStateGetter struct { sync.RWMutex - nodeState updatev1alpha1.CSPNodeState + nodeStates map[string]updatev1alpha1.CSPNodeState nodeStateErr error deleteNodeErr error } -func (g *stubNodeStateGetter) GetNodeState(_ context.Context, _ string) (updatev1alpha1.CSPNodeState, error) { +func (g *stubNodeStateGetter) GetNodeState(_ context.Context, providerID string) (updatev1alpha1.CSPNodeState, error) { g.RLock() defer g.RUnlock() - return g.nodeState, g.nodeStateErr + + if _, ok := g.nodeStates[providerID]; !ok { + panic("unexpected call to GetNodeState") + } + + return g.nodeStates[providerID], g.nodeStateErr } func (g *stubNodeStateGetter) DeleteNode(_ context.Context, _ string) error { @@ -250,8 +255,19 @@ func (g *stubNodeStateGetter) DeleteNode(_ context.Context, _ string) error { // thread safe methods to update the stub while in use -func (g *stubNodeStateGetter) setNodeState(nodeState updatev1alpha1.CSPNodeState) { +func (g *stubNodeStateGetter) setNodeState(providerID string, nodeState updatev1alpha1.CSPNodeState) { + g.Lock() + defer g.Unlock() + if g.nodeStates == nil { + g.nodeStates = make(map[string]updatev1alpha1.CSPNodeState) + } + g.nodeStates[providerID] = nodeState +} + +func (g *stubNodeStateGetter) reset() { g.Lock() defer g.Unlock() - g.nodeState = nodeState + g.nodeStates = nil + g.nodeStateErr = nil + g.deleteNodeErr = nil } diff --git a/operators/constellation-node-operator/controllers/suite_test.go b/operators/constellation-node-operator/controllers/suite_test.go index 7b4e232181..eafc162814 100644 --- a/operators/constellation-node-operator/controllers/suite_test.go +++ b/operators/constellation-node-operator/controllers/suite_test.go @@ -154,7 +154,7 @@ type fakeCollection struct { func (c *fakeCollection) reset() { c.scalingGroupUpdater.reset() - c.nodeStateGetter.setNodeState("") + c.nodeStateGetter.reset() c.nodeReplacer.reset() }