diff --git a/pkg/karmadactl/addons/descheduler/manifests.go b/pkg/karmadactl/addons/descheduler/manifests.go index 59e5755741ae..4560da0f3c69 100644 --- a/pkg/karmadactl/addons/descheduler/manifests.go +++ b/pkg/karmadactl/addons/descheduler/manifests.go @@ -44,7 +44,7 @@ spec: imagePullPolicy: IfNotPresent command: - /bin/karmada-descheduler - - --kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config - --metrics-bind-address=0.0.0.0:8080 - --health-probe-bind-address=0.0.0.0:10358 - --leader-elect-resource-namespace={{ .Namespace }} @@ -66,19 +66,18 @@ spec: name: metrics protocol: TCP volumeMounts: + - name: karmada-config + mountPath: /etc/karmada/config - name: k8s-certs mountPath: /etc/karmada/pki readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig volumes: + - name: karmada-config + secret: + secretName: karmada-descheduler-config - name: k8s-certs secret: secretName: karmada-cert - - name: kubeconfig - secret: - secretName: kubeconfig ` // DeploymentReplace is a struct to help to concrete diff --git a/pkg/karmadactl/addons/init/enable_option.go b/pkg/karmadactl/addons/init/enable_option.go index 9453af086a79..0c34a6aa1a66 100644 --- a/pkg/karmadactl/addons/init/enable_option.go +++ b/pkg/karmadactl/addons/init/enable_option.go @@ -26,7 +26,7 @@ import ( "k8s.io/klog/v2" "k8s.io/utils/strings/slices" - cmdinit "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" "github.com/karmada-io/karmada/pkg/version" ) @@ -148,10 +148,17 @@ func (o *CommandAddonsEnableOption) Validate(args []string) error { } secretClient := o.KubeClientSet.CoreV1().Secrets(o.Namespace) - _, err = secretClient.Get(context.TODO(), cmdinit.KubeConfigSecretAndMountName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return fmt.Errorf("secrets `kubeconfig` is not found in namespace %s, please execute karmadactl init to deploy karmada first", o.Namespace) + for _, addon := range getEnableAddons(args) { + if addon.Name == EstimatorResourceName { + // estimator not rely on karmada config secret + continue + } + karmadaConfigSecretName := addon.Name + options.KarmadaConfigSecretSuffix + _, err = secretClient.Get(context.TODO(), karmadaConfigSecretName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return fmt.Errorf("secrets `%s` is not found in namespace %s, please execute karmadactl init to deploy karmada first", karmadaConfigSecretName, o.Namespace) + } } } @@ -187,21 +194,8 @@ func (o *CommandAddonsEnableOption) Validate(args []string) error { // Run start enable Karmada addons func (o *CommandAddonsEnableOption) Run(args []string) error { - var enableAddons = map[string]*Addon{} - - // collect enabled addons - for _, item := range args { - if item == "all" { - enableAddons = Addons - break - } - if addon := Addons[item]; addon != nil { - enableAddons[item] = addon - } - } - // enable addons - for name, addon := range enableAddons { + for name, addon := range getEnableAddons(args) { klog.Infof("Start to enable addon %s", name) if err := addon.Enable(o); err != nil { klog.Errorf("Install addon %s failed", name) @@ -229,3 +223,20 @@ func validAddonNames(addonNames []string) error { } return nil } + +func getEnableAddons(addonNames []string) map[string]*Addon { + var enableAddons = map[string]*Addon{} + + // collect enabled addons + for _, item := range addonNames { + if item == "all" { + enableAddons = Addons + break + } + if addon := Addons[item]; addon != nil { + enableAddons[item] = addon + } + } + + return enableAddons +} diff --git a/pkg/karmadactl/addons/metricsadapter/manifests.go b/pkg/karmadactl/addons/metricsadapter/manifests.go index 607279bc5118..be297dc1c3e5 100644 --- a/pkg/karmadactl/addons/metricsadapter/manifests.go +++ b/pkg/karmadactl/addons/metricsadapter/manifests.go @@ -43,18 +43,11 @@ spec: - name: karmada-metrics-adapter image: {{ .Image }} imagePullPolicy: IfNotPresent - volumeMounts: - - name: k8s-certs - mountPath: /etc/karmada/pki - readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig command: - /bin/karmada-metrics-adapter - - --kubeconfig=/etc/kubeconfig - - --authentication-kubeconfig=/etc/kubeconfig - - --authorization-kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config + - --authentication-kubeconfig=/etc/karmada/config/karmada.config + - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --client-ca-file=/etc/karmada/pki/ca.crt - --audit-log-path=- - --audit-log-maxage=0 @@ -81,13 +74,19 @@ spec: resources: requests: cpu: 100m + volumeMounts: + - name: karmada-config + mountPath: /etc/karmada/config + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true volumes: + - name: karmada-config + secret: + secretName: karmada-metrics-adapter-config - name: k8s-certs secret: secretName: karmada-cert - - name: kubeconfig - secret: - secretName: kubeconfig ` karmadaMetricsAdapterService = ` diff --git a/pkg/karmadactl/addons/search/manifests.go b/pkg/karmadactl/addons/search/manifests.go index 23e84ed65cde..fa3ee5915fc6 100644 --- a/pkg/karmadactl/addons/search/manifests.go +++ b/pkg/karmadactl/addons/search/manifests.go @@ -43,18 +43,11 @@ spec: - name: karmada-search image: {{ .Image }} imagePullPolicy: IfNotPresent - volumeMounts: - - name: k8s-certs - mountPath: /etc/karmada/pki - readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig command: - /bin/karmada-search - - --kubeconfig=/etc/kubeconfig - - --authentication-kubeconfig=/etc/kubeconfig - - --authorization-kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config + - --authentication-kubeconfig=/etc/karmada/config/karmada.config + - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --etcd-servers={{ .ETCDSevers }} - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt - --etcd-certfile=/etc/karmada/pki/etcd-client.crt @@ -78,13 +71,19 @@ spec: resources: requests: cpu: 100m + volumeMounts: + - name: karmada-config + mountPath: /etc/karmada/config + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true volumes: + - name: karmada-config + secret: + secretName: karmada-search-config - name: k8s-certs secret: secretName: karmada-cert - - name: kubeconfig - secret: - secretName: kubeconfig ` karmadaSearchService = ` diff --git a/pkg/karmadactl/cmdinit/kubernetes/deploy.go b/pkg/karmadactl/cmdinit/kubernetes/deploy.go index cad1a9fce8af..33f99f5f76b6 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deploy.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deploy.go @@ -63,6 +63,17 @@ var ( options.FrontProxyClientCertAndKeyName, } + karmadaConfigList = []string{ + options.KarmadaAggregatedApiserverConfig, + options.KarmadaControllerManagerConfig, + options.KubeControllerManagerConfig, + options.KarmadaSchedulerConfig, + options.KarmadaDeschedulerConfig, + options.KarmadaMetricsAdapterConfig, + options.KarmadaSearchConfig, + options.KarmadaWebhookConfig, + } + emptyByteSlice = make([]byte, 0) externalEtcdCertSpecialization = map[string]func(*CommandInitOption) ([]byte, []byte, error){ options.EtcdCaCertAndKeyName: func(option *CommandInitOption) (cert, key []byte, err error) { @@ -397,7 +408,7 @@ func (i *CommandInitOption) prepareCRD() error { } func (i *CommandInitOption) createCertsSecrets() error { - // Create kubeconfig Secret + // 1. Create karmada-config Secret karmadaServerURL := fmt.Sprintf("https://%s.%s.svc.%s:%v", karmadaAPIServerDeploymentAndServiceName, i.Namespace, i.HostClusterDomain, karmadaAPIServerContainerPort) config := utils.CreateWithCerts(karmadaServerURL, options.UserName, options.UserName, i.CertAndKeyFileData[fmt.Sprintf("%s.crt", globaloptions.CaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.KarmadaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.KarmadaCertAndKeyName)]) @@ -406,10 +417,13 @@ func (i *CommandInitOption) createCertsSecrets() error { return fmt.Errorf("failure while serializing admin kubeConfig. %v", err) } - kubeConfigSecret := i.SecretFromSpec(KubeConfigSecretAndMountName, corev1.SecretTypeOpaque, map[string]string{KubeConfigSecretAndMountName: string(configBytes)}) - if err = util.CreateOrUpdateSecret(i.KubeClientSet, kubeConfigSecret); err != nil { - return err + for _, karmadaConfigSecretName := range karmadaConfigList { + kubeConfigSecret := i.SecretFromSpec(karmadaConfigSecretName, corev1.SecretTypeOpaque, map[string]string{options.KarmadaConfigFieldName: string(configBytes)}) + if err = util.CreateOrUpdateSecret(i.KubeClientSet, kubeConfigSecret); err != nil { + return err + } } + // Create certs Secret etcdCert := map[string]string{ fmt.Sprintf("%s.crt", options.EtcdCaCertAndKeyName): string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.EtcdCaCertAndKeyName)]), diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index 835ec04c263a..e97eec6f167b 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -18,6 +18,7 @@ package kubernetes import ( "fmt" + "path/filepath" "strings" appsv1 "k8s.io/api/apps/v1" @@ -38,10 +39,9 @@ const ( metricsPortName = "metrics" defaultMetricsPort = 8080 - // KubeConfigSecretAndMountName is the secret and volume mount name of karmada kubeconfig - KubeConfigSecretAndMountName = "kubeconfig" karmadaCertsVolumeMountPath = "/etc/karmada/pki" - kubeConfigContainerMountPath = "/etc/kubeconfig" + karmadaConfigVolumeName = "karmada-config" + karmadaConfigVolumeMountPath = "/etc/karmada/config" karmadaAPIServerDeploymentAndServiceName = "karmada-apiserver" karmadaAPIServerContainerPort = 5443 serviceClusterIP = "10.96.0.0/12" @@ -303,8 +303,9 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 Command: []string{ "kube-controller-manager", "--allocate-node-cidrs=true", - "--authentication-kubeconfig=/etc/kubeconfig", - "--authorization-kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), + fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), + fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), "--bind-address=0.0.0.0", fmt.Sprintf("--client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), "--cluster-cidr=10.244.0.0/16", @@ -312,7 +313,6 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 fmt.Sprintf("--cluster-signing-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), fmt.Sprintf("--cluster-signing-key-file=%s/%s.key", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), "--controllers=namespace,garbagecollector,serviceaccount-token,ttl-after-finished,bootstrapsigner,tokencleaner,csrapproving,csrcleaner,csrsigning,clusterrole-aggregation", - "--kubeconfig=/etc/kubeconfig", "--leader-elect=true", fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace), "--node-cidr-mask-size=24", @@ -332,10 +332,9 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, { Name: globaloptions.KarmadaCertsName, @@ -347,10 +346,10 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KubeControllerManagerConfig, }, }, }, @@ -448,7 +447,7 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment ImagePullPolicy: corev1.PullPolicy(i.ImagePullPolicy), Command: []string{ "/bin/karmada-scheduler", - "--kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), "--metrics-bind-address=0.0.0.0:8080", "--health-probe-bind-address=0.0.0.0:10351", "--enable-scheduler-estimator=true", @@ -469,10 +468,9 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, { Name: globaloptions.KarmadaCertsName, @@ -484,10 +482,10 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KarmadaSchedulerConfig, }, }, }, @@ -587,7 +585,7 @@ func (i *CommandInitOption) makeKarmadaControllerManagerDeployment() *appsv1.Dep ImagePullPolicy: corev1.PullPolicy(i.ImagePullPolicy), Command: []string{ "/bin/karmada-controller-manager", - "--kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), "--metrics-bind-address=:8080", "--health-probe-bind-address=0.0.0.0:10357", "--cluster-status-update-frequency=10s", @@ -609,20 +607,19 @@ func (i *CommandInitOption) makeKarmadaControllerManagerDeployment() *appsv1.Dep }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, }, }, }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KarmadaControllerManagerConfig, }, }, }, @@ -711,7 +708,7 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { ImagePullPolicy: corev1.PullPolicy(i.ImagePullPolicy), Command: []string{ "/bin/karmada-webhook", - "--kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), "--bind-address=0.0.0.0", "--metrics-bind-address=:8080", fmt.Sprintf("--secure-port=%v", webhookTargetPort), @@ -732,10 +729,9 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, { Name: webhookCertsName, @@ -748,10 +744,10 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KarmadaWebhookConfig, }, }, }, @@ -842,9 +838,9 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D } command := []string{ "/bin/karmada-aggregated-apiserver", - "--kubeconfig=/etc/kubeconfig", - "--authentication-kubeconfig=/etc/kubeconfig", - "--authorization-kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), + fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), + fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), fmt.Sprintf("--etcd-servers=%s", etcdServers), fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), @@ -886,12 +882,18 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D Image: i.karmadaAggregatedAPIServerImage(), ImagePullPolicy: corev1.PullPolicy(i.ImagePullPolicy), Command: command, + ReadinessProbe: readinesProbe, + LivenessProbe: livenesProbe, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, { Name: globaloptions.KarmadaCertsName, @@ -899,21 +901,14 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D MountPath: karmadaCertsVolumeMountPath, }, }, - ReadinessProbe: readinesProbe, - LivenessProbe: livenesProbe, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - }, - }, }, }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KarmadaAggregatedApiserverConfig, }, }, }, diff --git a/pkg/karmadactl/cmdinit/options/global.go b/pkg/karmadactl/cmdinit/options/global.go index 9d132cea0583..7e44343b760b 100644 --- a/pkg/karmadactl/cmdinit/options/global.go +++ b/pkg/karmadactl/cmdinit/options/global.go @@ -39,4 +39,25 @@ const ( KarmadaKubeConfigName = "karmada-apiserver.config" // WaitComponentReadyTimeout wait component ready time WaitComponentReadyTimeout = 120 + + // KarmadaConfigSecretSuffix karmada config secret suffix + KarmadaConfigSecretSuffix = "-config" //nolint:gosec + // KarmadaConfigFieldName the field stores karmada config in karmada config secret + KarmadaConfigFieldName = "karmada.config" //nolint:gosec + // KarmadaAggregatedApiserverConfig karmada config of karmada-aggregated-apiserver + KarmadaAggregatedApiserverConfig = "karmada-aggregated-apiserver" + KarmadaConfigSecretSuffix + // KarmadaControllerManagerConfig karmada config of karmada-controller-manager + KarmadaControllerManagerConfig = "karmada-controller-manager" + KarmadaConfigSecretSuffix + // KubeControllerManagerConfig karmada config of kube-controller-manager + KubeControllerManagerConfig = "kube-controller-manager" + KarmadaConfigSecretSuffix + // KarmadaSchedulerConfig karmada config of karmada-scheduler + KarmadaSchedulerConfig = "karmada-scheduler" + KarmadaConfigSecretSuffix + // KarmadaDeschedulerConfig karmada config of karmada-descheduler + KarmadaDeschedulerConfig = "karmada-descheduler" + KarmadaConfigSecretSuffix + // KarmadaMetricsAdapterConfig karmada config of karmada-metrics-adapter + KarmadaMetricsAdapterConfig = "karmada-metrics-adapter" + KarmadaConfigSecretSuffix + // KarmadaSearchConfig karmada config of karmada-search + KarmadaSearchConfig = "karmada-search" + KarmadaConfigSecretSuffix + // KarmadaWebhookConfig karmada config of karmada-webhook + KarmadaWebhookConfig = "karmada-webhook" + KarmadaConfigSecretSuffix )