diff --git a/.changelog/1442.txt b/.changelog/1442.txt new file mode 100644 index 000000000..225eadfe1 --- /dev/null +++ b/.changelog/1442.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +`resource/helm_release`: Add `resources` map attribute to allow drift detection against live kubernetes resources +``` diff --git a/go.mod b/go.mod index c9cdbb351..839a779db 100644 --- a/go.mod +++ b/go.mod @@ -17,8 +17,12 @@ require ( helm.sh/helm/v3 v3.15.3 k8s.io/api v0.30.3 k8s.io/apimachinery v0.30.3 + k8s.io/cli-runtime v0.30.3 k8s.io/client-go v0.30.3 k8s.io/klog v1.0.0 + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 + k8s.io/kubectl v0.30.3 + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 sigs.k8s.io/yaml v1.4.0 ) @@ -58,8 +62,10 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect + github.com/fatih/camelcase v1.0.0 // indirect github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/fvbommel/sortorder v1.1.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect @@ -102,6 +108,7 @@ require ( github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.0 // indirect @@ -179,15 +186,11 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.0 // indirect k8s.io/apiserver v0.30.0 // indirect - k8s.io/cli-runtime v0.30.0 // indirect - k8s.io/component-base v0.30.0 // indirect + k8s.io/component-base v0.30.3 // indirect k8s.io/klog/v2 v2.120.1 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/kubectl v0.30.0 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect oras.land/oras-go v1.2.5 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index 98fdc9bb9..c54a245d3 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= @@ -129,6 +131,8 @@ github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6 github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw= +github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -293,6 +297,8 @@ github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgf github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -680,20 +686,20 @@ k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/apiserver v0.30.0 h1:QCec+U72tMQ+9tR6A0sMBB5Vh6ImCEkoKkTDRABWq6M= k8s.io/apiserver v0.30.0/go.mod h1:smOIBq8t0MbKZi7O7SyIpjPsiKJ8qa+llcFCluKyqiY= -k8s.io/cli-runtime v0.30.0 h1:0vn6/XhOvn1RJ2KJOC6IRR2CGqrpT6QQF4+8pYpWQ48= -k8s.io/cli-runtime v0.30.0/go.mod h1:vATpDMATVTMA79sZ0YUCzlMelf6rUjoBzlp+RnoM+cg= +k8s.io/cli-runtime v0.30.3 h1:aG69oRzJuP2Q4o8dm+f5WJIX4ZBEwrvdID0+MXyUY6k= +k8s.io/cli-runtime v0.30.3/go.mod h1:hwrrRdd9P84CXSKzhHxrOivAR9BRnkMt0OeP5mj7X30= k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= -k8s.io/component-base v0.30.0 h1:cj6bp38g0ainlfYtaOQuRELh5KSYjhKxM+io7AUIk4o= -k8s.io/component-base v0.30.0/go.mod h1:V9x/0ePFNaKeKYA3bOvIbrNoluTSG+fSJKjLdjOoeXQ= +k8s.io/component-base v0.30.3 h1:Ci0UqKWf4oiwy8hr1+E3dsnliKnkMLZMVbWzeorlk7s= +k8s.io/component-base v0.30.3/go.mod h1:C1SshT3rGPCuNtBs14RmVD2xW0EhRSeLvBh7AGk1quA= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/kubectl v0.30.0 h1:xbPvzagbJ6RNYVMVuiHArC1grrV5vSmmIcSZuCdzRyk= -k8s.io/kubectl v0.30.0/go.mod h1:zgolRw2MQXLPwmic2l/+iHs239L49fhSeICuMhQQXTI= +k8s.io/kubectl v0.30.3 h1:YIBBvMdTW0xcDpmrOBzcpUVsn+zOgjMYIu7kAq+yqiI= +k8s.io/kubectl v0.30.3/go.mod h1:IcR0I9RN2+zzTRUa1BzZCm4oM0NLOawE6RzlDvd1Fpo= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= diff --git a/helm/kube_resources.go b/helm/kube_resources.go new file mode 100644 index 000000000..ff26c1a27 --- /dev/null +++ b/helm/kube_resources.go @@ -0,0 +1,224 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package helm + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/kube" + "helm.sh/helm/v3/pkg/release" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/managedfields" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/discovery" + "k8s.io/kube-openapi/pkg/util/proto" + "k8s.io/kubectl/pkg/cmd/diff" + "k8s.io/kubectl/pkg/scheme" + "sigs.k8s.io/structured-merge-diff/v4/fieldpath" +) + +func getKubeClient(actionConfig *action.Configuration) (*kube.Client, error) { + kc, ok := actionConfig.KubeClient.(*kube.Client) + if !ok { + return nil, errors.Errorf("client is not a *kube.Client") + } + return kc, nil +} + +// regenerateGVKParser builds the parser from the raw OpenAPI schema. +func regenerateGVKParser(dc discovery.DiscoveryInterface) (*managedfields.GvkParser, error) { + doc, err := dc.OpenAPISchema() + if err != nil { + return nil, err + } + + models, err := proto.NewOpenAPIData(doc) + if err != nil { + return nil, err + } + + return managedfields.NewGVKParser(models, false) +} + +// removeUnmanagedFields removes fields updated by `kube-controller-manager` or +// through subresource apis from a kubernetes object +func removeUnmanagedFields(parser *managedfields.GvkParser, obj runtime.Object, gvk schema.GroupVersionKind) error { + parseableType := parser.Type(gvk) + if parseableType == nil { + return errors.Errorf("no parseable type found for %s", gvk.String()) + } + typedObj, err := parseableType.FromStructured(obj) + if err != nil { + return err + } + accessor, err := apimeta.Accessor(obj) + if err != nil { + return err + } + objManagedFields := accessor.GetManagedFields() + fieldSet := &fieldpath.Set{} + for _, mf := range objManagedFields { + if mf.Manager == "kube-controller-manager" || mf.Subresource != "" { + fs := &fieldpath.Set{} + if err := fs.FromJSON(bytes.NewReader(mf.FieldsV1.Raw)); err != nil { + return err + } + fieldSet = fieldSet.Union(fs) + } + } + u := typedObj.RemoveItems(fieldSet).AsValue().Unstructured() + m, ok := u.(map[string]interface{}) + if !ok { + return errors.Errorf("unexpected type %T", u) + } + return runtime.DefaultUnstructuredConverter.FromUnstructured(m, obj) +} + +// mapRuntimeObjects maps a list of kubernetes objects by key to their JSON +// representation, with unmanaged fields removed and sensitive values redacted +func mapRuntimeObjects(kc *kube.Client, objects []runtime.Object, d resourceGetter) (map[string]string, error) { + clientSet, err := kc.Factory.KubernetesClientSet() + if err != nil { + return nil, err + } + parser, err := regenerateGVKParser(clientSet.Discovery()) + if err != nil { + return nil, err + } + + mappedObjects := make(map[string]string) + for _, obj := range objects { + gvk := obj.GetObjectKind().GroupVersionKind() + if gvk.Kind == "Secret" { + secret := &corev1.Secret{} + err := scheme.Scheme.Convert(obj, secret, nil) + if err != nil { + return nil, err + } + redactSecretData(secret) + obj = secret + } + accessor, err := apimeta.Accessor(obj) + if err != nil { + return nil, err + } + key := fmt.Sprintf("%s/%s/%s/%s", + strings.ToLower(gvk.GroupKind().String()), + gvk.Version, + accessor.GetNamespace(), + accessor.GetName(), + ) + if err := removeUnmanagedFields(parser, obj, gvk); err != nil { + return nil, err + } + accessor.SetUID(types.UID("")) + accessor.SetCreationTimestamp(metav1.Time{}) + accessor.SetResourceVersion("") + accessor.SetManagedFields(nil) + objJSON, err := json.Marshal(obj) + if err != nil { + return nil, err + } + mappedObjects[key] = redactSensitiveValues(string(objJSON), d) + } + return mappedObjects, nil +} + +func mapResources(actionConfig *action.Configuration, r *release.Release, d resourceGetter, f func(*resource.Info) (runtime.Object, error)) (map[string]string, error) { + resources, err := actionConfig.KubeClient.Build(bytes.NewBufferString(r.Manifest), false) + if err != nil { + return nil, err + } + var objects []runtime.Object + err = resources.Visit(func(i *resource.Info, err error) error { + if err != nil { + return err + } + obj, err := f(i) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + objects = append(objects, obj) + return nil + }) + if err != nil { + return nil, err + } + kc, err := getKubeClient(actionConfig) + if err != nil { + return nil, err + } + return mapRuntimeObjects(kc, objects, d) +} + +// getLiveResources gets the live kubernetes resources of a release +func getLiveResources(r *release.Release, m *Meta, d resourceGetter) (map[string]string, error) { + actionConfig, err := m.GetHelmConfiguration(r.Namespace) + if err != nil { + return nil, err + } + kc, err := getKubeClient(actionConfig) + if err != nil { + return nil, err + } + return mapResources(actionConfig, r, d, func(i *resource.Info) (runtime.Object, error) { + gvk := i.Object.GetObjectKind().GroupVersionKind() + return kc.Factory.NewBuilder(). + Unstructured(). + NamespaceParam(i.Namespace).DefaultNamespace(). + ResourceNames(gvk.GroupKind().String(), i.Name). + Flatten(). + Do(). + Object() + }) +} + +// getDryRunResources gets the kubernetes resources as they would look like if +// the helm manifest is applied to the cluster. this is useful for detecting the +// differences between the live cluster state and the desired state. +func getDryRunResources(r *release.Release, m *Meta, d resourceGetter) (map[string]string, error) { + actionConfig, err := m.GetHelmConfiguration(r.Namespace) + if err != nil { + return nil, err + } + ioStreams := genericiooptions.IOStreams{ + In: os.Stdin, + Out: os.Stdout, + ErrOut: os.Stderr, + } + fieldManager := "terraform-provider-helm" + if os.Args[0] != "" { + fieldManager = filepath.Base(os.Args[0]) + } + return mapResources(actionConfig, r, d, func(i *resource.Info) (runtime.Object, error) { + info := &diff.InfoObject{ + LocalObj: i.Object, + Info: i, + Encoder: scheme.DefaultJSONEncoder(), + Force: false, + ServerSideApply: true, + FieldManager: fieldManager, + ForceConflicts: true, + IOStreams: ioStreams, + } + return info.Merged() + }) +} diff --git a/helm/manifest_json.go b/helm/manifest_json.go index ffd6c62c1..30bdf67be 100644 --- a/helm/manifest_json.go +++ b/helm/manifest_json.go @@ -56,10 +56,7 @@ func convertYAMLManifestToJSON(manifest string) (string, error) { return "", err } - for k, v := range secret.Data { - h := hashSensitiveValue(string(v)) - secret.Data[k] = []byte(h) - } + redactSecretData(&secret) jsonbytes, err = json.Marshal(secret) if err != nil { @@ -89,6 +86,14 @@ func hashSensitiveValue(v string) string { return fmt.Sprintf("(sensitive value %x)", hash) } +// redactSecretData redacts the `Data` of a Secret +func redactSecretData(secret *corev1.Secret) { + for k, v := range secret.Data { + h := hashSensitiveValue(string(v)) + secret.Data[k] = []byte(h) + } +} + // redactSensitiveValues removes values that appear in `set_sensitive` blocks from // the manifest JSON func redactSensitiveValues(text string, d resourceGetter) string { diff --git a/helm/resource_release.go b/helm/resource_release.go index 8b0299be7..9d83b1f3e 100644 --- a/helm/resource_release.go +++ b/helm/resource_release.go @@ -386,6 +386,16 @@ func resourceRelease() *schema.Resource { Description: "The rendered manifest as JSON.", Computed: true, }, + "resources": { + Type: schema.TypeMap, + Computed: true, + Description: "The kubernetes resources created by this release.", + Elem: &schema.Schema{ + Type: schema.TypeString, + Computed: true, + Description: "The information of a kubernetes resource as JSON.", + }, + }, "upgrade_install": { Type: schema.TypeBool, Optional: true, @@ -895,6 +905,11 @@ func resourceDiff(ctx context.Context, d *schema.ResourceDiff, meta interface{}) debug("%s upgrade_install is enabled: %t", logID, enableUpgradeStrategy) debug("%s targetVersion for release %s: '%s'", logID, name, targetVersion) + if !m.ExperimentEnabled("manifest") { + d.Clear("manifest") + d.Clear("resources") + } + actionConfig, err := m.GetHelmConfiguration(namespace) if err != nil { return err @@ -995,6 +1010,7 @@ func resourceDiff(ctx context.Context, d *schema.ResourceDiff, meta interface{}) // but this is not possible with the SDK debug("not all values are known, skipping dry run to render manifest") d.SetNewComputed("manifest") + d.SetNewComputed("resources") return d.SetNewComputed("version") } @@ -1053,8 +1069,9 @@ func resourceDiff(ctx context.Context, d *schema.ResourceDiff, meta interface{}) // NOTE it would be nice to return a diagnostic here to warn the user // that we can't generate the diff here because the cluster is not yet // reachable but this is not supported by CustomizeDiffFunc - debug(`cluster was unreachable at create time, marking "manifest" as computed`) - return d.SetNewComputed("manifest") + debug(`cluster was unreachable at create time, marking "manifest" and "resources" as computed`) + d.SetNewComputed("manifest") + return d.SetNewComputed("resources") } return err } @@ -1064,6 +1081,9 @@ func resourceDiff(ctx context.Context, d *schema.ResourceDiff, meta interface{}) return err } manifest := redactSensitiveValues(string(jsonManifest), d) + + d.SetNewComputed("resources") + return d.SetNew("manifest", manifest) } @@ -1074,6 +1094,7 @@ func resourceDiff(ctx context.Context, d *schema.ResourceDiff, meta interface{}) return d.SetNew("version", chart.Metadata.Version) } d.SetNewComputed("manifest") + d.SetNewComputed("resources") return d.SetNewComputed("version") } else if err != nil { return fmt.Errorf("error retrieving old release for a diff: %v", err) @@ -1112,6 +1133,7 @@ func resourceDiff(ctx context.Context, d *schema.ResourceDiff, meta interface{}) } d.SetNewComputed("version") d.SetNewComputed("manifest") + d.SetNewComputed("resources") return nil } else if err != nil { return fmt.Errorf("error running dry run for a diff: %v", err) @@ -1124,8 +1146,12 @@ func resourceDiff(ctx context.Context, d *schema.ResourceDiff, meta interface{}) manifest := redactSensitiveValues(string(jsonManifest), d) d.SetNew("manifest", manifest) debug("%s set manifest: %s", logID, jsonManifest) - } else { - d.Clear("manifest") + resources, err := getDryRunResources(dry, m, d) + if err != nil { + return err + } + d.SetNew("resources", resources) + debug("%s set resources: %v", logID, resources) } // handle possible upgrade_install scenarios when the version attribute is empty @@ -1193,6 +1219,11 @@ func setReleaseAttributes(d *schema.ResourceData, r *release.Release, meta inter } manifest := redactSensitiveValues(string(jsonManifest), d) d.Set("manifest", manifest) + resources, err := getLiveResources(r, m, d) + if err != nil { + return err + } + d.Set("resources", resources) } return d.Set("metadata", []map[string]interface{}{{ diff --git a/helm/resource_release_test.go b/helm/resource_release_test.go index 01a2c52d6..57337357a 100644 --- a/helm/resource_release_test.go +++ b/helm/resource_release_test.go @@ -4,7 +4,9 @@ package helm import ( + "bytes" "context" + "encoding/json" "fmt" "os" "os/exec" @@ -26,10 +28,17 @@ import ( "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/helmpath" + "helm.sh/helm/v3/pkg/kube" "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/releaseutil" "helm.sh/helm/v3/pkg/repo" + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + runtimeresource "k8s.io/cli-runtime/pkg/resource" _ "k8s.io/client-go/plugin/pkg/client/auth" ) @@ -1772,6 +1781,65 @@ func getReleaseJSONManifest(namespace, name string) (string, error) { return jsonManifest, nil } +func getTestKubeClient(namespace string) (*kube.Client, error) { + m := testAccProvider.Meta() + if m == nil { + return nil, fmt.Errorf("provider not properly initialized") + } + actionConfig, err := m.(*Meta).GetHelmConfiguration(namespace) + if err != nil { + return nil, err + } + return getKubeClient(actionConfig) +} + +func getReleaseJSONResources(namespace, name string) (map[string]string, error) { + kc, err := getTestKubeClient(namespace) + if err != nil { + return nil, err + } + + cmd := exec.Command("helm", "--kubeconfig", os.Getenv("KUBE_CONFIG_PATH"), "get", "manifest", "--namespace", namespace, name) + manifest, err := cmd.Output() + if err != nil { + return nil, err + } + + resources, err := kc.Build(bytes.NewBuffer(manifest), false) + if err != nil { + return nil, err + } + + var objects []runtime.Object + err = resources.Visit(func(i *runtimeresource.Info, err error) error { + if err != nil { + return err + } + gvk := i.Object.GetObjectKind().GroupVersionKind() + obj, err := kc.Factory.NewBuilder(). + Unstructured(). + NamespaceParam(i.Namespace).DefaultNamespace(). + ResourceNames(gvk.GroupKind().String(), i.Name). + Flatten(). + Do(). + Object() + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + objects = append(objects, obj) + return nil + }) + if err != nil { + return nil, err + } + + d := resourceRelease().Data(nil) + return mapRuntimeObjects(kc, objects, d) +} + func TestAccResourceRelease_manifest(t *testing.T) { name := randName("diff") namespace := createRandomNamespace(t) @@ -1840,6 +1908,170 @@ func TestAccResourceRelease_manifestUnknownValues(t *testing.T) { }) } +func patchDeployment(t *testing.T, namespace, name string, patchBytes []byte) func() { + return func() { + kc, err := getTestKubeClient(namespace) + if err != nil { + t.Fatal(err) + } + client, err := kc.Factory.KubernetesClientSet() + if err != nil { + t.Fatal(err) + } + + _, err = client.AppsV1().Deployments(namespace).Patch( + context.Background(), name, types.StrategicMergePatchType, + patchBytes, metav1.PatchOptions{}) + if err != nil { + t.Fatal(err) + } + for { + dep, err := client.AppsV1().Deployments(namespace).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + desiredReplicas := *dep.Spec.Replicas + if dep.Status.Replicas == desiredReplicas && dep.Status.AvailableReplicas == desiredReplicas && dep.Status.UpdatedReplicas == desiredReplicas { + break + } + time.Sleep(10 * time.Second) + } + } +} + +func checkResourceAttrMap(name, key string, expected map[string]string) resource.TestCheckFunc { + checks := []resource.TestCheckFunc{ + resource.TestCheckResourceAttr(name, fmt.Sprintf("%s.%%", key), strconv.Itoa(len(expected))), + } + for k, v := range expected { + checks = append(checks, resource.TestCheckResourceAttr(name, fmt.Sprintf("%s.%s", key, k), v)) + } + return resource.ComposeAggregateTestCheckFunc(checks...) +} + +func checkDeploymentReplicasAndGeneration(name, namespace, deploymentName string, replicas int32, generation int64) resource.TestCheckFunc { + deploymentKey := fmt.Sprintf("resources.deployment.apps/v1/%s/%s", namespace, deploymentName) + return resource.TestCheckResourceAttrWith(name, deploymentKey, func(value string) error { + var deployment appsv1.Deployment + if err := json.Unmarshal([]byte(value), &deployment); err != nil { + return err + } + if deployment.Spec.Replicas == nil { + return fmt.Errorf("expected replicas to be set") + } + if *deployment.Spec.Replicas != replicas { + return fmt.Errorf("expected replicas to be %d, got %d", replicas, *deployment.Spec.Replicas) + } + if deployment.Generation != generation { + return fmt.Errorf("expected generation to be %d, got %d", generation, deployment.Generation) + } + return nil + }) +} + +func TestAccResourceRelease_manifestServerDiff(t *testing.T) { + name := randName("serverdiff") + namespace := createRandomNamespace(t) + defer deleteNamespace(t, namespace) + + config := testAccHelmReleaseConfigManifestExperimentEnabled(testResourceName, namespace, name, "1.2.3") + + fullName := fmt.Sprintf("%s-test-chart", name) + foregroundPropagation := metav1.DeletePropagationForeground + + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheck(t) + }, + ProviderFactories: map[string]func() (*schema.Provider, error){ + "helm": func() (*schema.Provider, error) { + return Provider(), nil + }, + }, + CheckDestroy: testAccCheckHelmReleaseDestroy(namespace), + Steps: []resource.TestStep{ + { + Config: config, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr("helm_release.test", "metadata.0.name", name), + resource.TestCheckResourceAttr("helm_release.test", "metadata.0.namespace", namespace), + resource.TestCheckResourceAttr("helm_release.test", "metadata.0.version", "1.2.3"), + func(state *terraform.State) error { + t.Logf("getting JSON server resources for release %q", name) + r, err := getReleaseJSONResources(namespace, name) + if err != nil { + t.Fatal(err) + } + return checkResourceAttrMap("helm_release.test", "resources", r)(state) + }, + checkDeploymentReplicasAndGeneration("helm_release.test", namespace, fullName, 1, 1), + ), + }, + { + // patch the deployment to have 2 replicas (generation 2) then apply the + // config to restore the replicas to 1 (generation 3) + PreConfig: patchDeployment(t, namespace, fullName, []byte(`{"spec":{"replicas":2}}`)), + Config: config, + Check: checkDeploymentReplicasAndGeneration("helm_release.test", namespace, fullName, 1, 3), + }, + { + // patch the deployment to have 2 replicas (generation 4) then apply a + // new config to set the replicas to 3 (generation 5) + PreConfig: patchDeployment(t, namespace, fullName, []byte(`{"spec":{"replicas":2}}`)), + Config: testAccHelmReleaseConfigManifestExperimentEnabledSetReplicas(testResourceName, namespace, name, "1.2.3"), + Check: checkDeploymentReplicasAndGeneration("helm_release.test", namespace, fullName, 3, 5), + }, + { + // patch the deployment to have 1 replicas (generation 6) then apply the + // original config (without `replicaCount` set) which sets the replicas + // to the previous value of 3 (generation 7) + PreConfig: patchDeployment(t, namespace, fullName, []byte(`{"spec":{"replicas":1}}`)), + Config: config, + Check: checkDeploymentReplicasAndGeneration("helm_release.test", namespace, fullName, 3, 7), + }, + { + // patch the deployment to have 1 replicas (generation 8) then apply the + // original config but now with `reset_values = true` so the `replicaCount` + // is reset to the chart's default value of 1 (generation 8, no changes) + PreConfig: patchDeployment(t, namespace, fullName, []byte(`{"spec":{"replicas":1}}`)), + Config: testAccHelmReleaseConfigManifestExperimentEnabledResetValues(testResourceName, namespace, name, "1.2.3"), + Check: checkDeploymentReplicasAndGeneration("helm_release.test", namespace, fullName, 1, 8), + }, + { + // delete the service then apply the previous config to recreate it + PreConfig: func() { + kc, err := getTestKubeClient(namespace) + if err != nil { + t.Fatal(err) + } + client, err := kc.Factory.KubernetesClientSet() + if err != nil { + t.Fatal(err) + } + err = client.CoreV1().Services(namespace).Delete(context.Background(), fullName, metav1.DeleteOptions{ + PropagationPolicy: &foregroundPropagation, + }) + if err != nil { + t.Fatal(err) + } + }, + Config: testAccHelmReleaseConfigManifestExperimentEnabledResetValues(testResourceName, namespace, name, "1.2.3"), + Check: resource.ComposeAggregateTestCheckFunc( + checkResourceAttrExists("helm_release.test", fmt.Sprintf("resources.service/v1/%s/%s", namespace, fullName)), + func(state *terraform.State) error { + t.Logf("getting JSON server resources for release %q", name) + r, err := getReleaseJSONResources(namespace, name) + if err != nil { + t.Fatal(err) + } + return checkResourceAttrMap("helm_release.test", "resources", r)(state) + }, + ), + }, + }, + }) +} + func TestAccResourceRelease_set_list_chart(t *testing.T) { name := randName("helm-setlist-chart") namespace := createRandomNamespace(t) @@ -2280,6 +2512,46 @@ func testAccHelmReleaseConfigManifestExperimentEnabled(resource, ns, name, versi `, resource, name, ns, testRepositoryURL, version) } +func testAccHelmReleaseConfigManifestExperimentEnabledResetValues(resource, ns, name, version string) string { + return fmt.Sprintf(` + provider helm { + experiments { + manifest = true + } + } + resource "helm_release" "%s" { + name = %q + namespace = %q + repository = %q + version = %q + chart = "test-chart" + reset_values = true + } + `, resource, name, ns, testRepositoryURL, version) +} + +func testAccHelmReleaseConfigManifestExperimentEnabledSetReplicas(resource, ns, name, version string) string { + return fmt.Sprintf(` + provider helm { + experiments { + manifest = true + } + } + resource "helm_release" "%s" { + name = %q + namespace = %q + repository = %q + version = %q + chart = "test-chart" + + set { + name = "replicaCount" + value = 3 + } + } + `, resource, name, ns, testRepositoryURL, version) +} + func testAccHelmReleaseConfigManifestUnknownValues(resource, ns, name, version string) string { return fmt.Sprintf(` provider helm {