Detect drifts between the manifest and its live Kubernetes resources

go.mod

@@ -17,8 +17,12 @@ require (
 	helm.sh/helm/v3 v3.15.3
 	k8s.io/api v0.30.3
 	k8s.io/apimachinery v0.30.3
+	k8s.io/cli-runtime v0.30.3
 	k8s.io/client-go v0.30.3
 	k8s.io/klog v1.0.0
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340
+	k8s.io/kubectl v0.30.3
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -58,8 +62,10 @@ require (
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
+	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/fvbommel/sortorder v1.1.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
@@ -102,6 +108,7 @@ require (
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmoiron/sqlx v1.3.5 // indirect
+	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.16.0 // indirect
@@ -179,15 +186,11 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.30.0 // indirect
 	k8s.io/apiserver v0.30.0 // indirect
-	k8s.io/cli-runtime v0.30.0 // indirect
-	k8s.io/component-base v0.30.0 // indirect
+	k8s.io/component-base v0.30.3 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
-	k8s.io/kubectl v0.30.0 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

go.sum

@@ -120,6 +120,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ
 github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
+github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
@@ -129,6 +131,8 @@ github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6
 github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
+github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw=
+github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -293,6 +297,8 @@ github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgf
 github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -680,20 +686,20 @@ k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc=
 k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
 k8s.io/apiserver v0.30.0 h1:QCec+U72tMQ+9tR6A0sMBB5Vh6ImCEkoKkTDRABWq6M=
 k8s.io/apiserver v0.30.0/go.mod h1:smOIBq8t0MbKZi7O7SyIpjPsiKJ8qa+llcFCluKyqiY=
-k8s.io/cli-runtime v0.30.0 h1:0vn6/XhOvn1RJ2KJOC6IRR2CGqrpT6QQF4+8pYpWQ48=
-k8s.io/cli-runtime v0.30.0/go.mod h1:vATpDMATVTMA79sZ0YUCzlMelf6rUjoBzlp+RnoM+cg=
+k8s.io/cli-runtime v0.30.3 h1:aG69oRzJuP2Q4o8dm+f5WJIX4ZBEwrvdID0+MXyUY6k=
+k8s.io/cli-runtime v0.30.3/go.mod h1:hwrrRdd9P84CXSKzhHxrOivAR9BRnkMt0OeP5mj7X30=
 k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=
 k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U=
-k8s.io/component-base v0.30.0 h1:cj6bp38g0ainlfYtaOQuRELh5KSYjhKxM+io7AUIk4o=
-k8s.io/component-base v0.30.0/go.mod h1:V9x/0ePFNaKeKYA3bOvIbrNoluTSG+fSJKjLdjOoeXQ=
+k8s.io/component-base v0.30.3 h1:Ci0UqKWf4oiwy8hr1+E3dsnliKnkMLZMVbWzeorlk7s=
+k8s.io/component-base v0.30.3/go.mod h1:C1SshT3rGPCuNtBs14RmVD2xW0EhRSeLvBh7AGk1quA=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
 k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
-k8s.io/kubectl v0.30.0 h1:xbPvzagbJ6RNYVMVuiHArC1grrV5vSmmIcSZuCdzRyk=
-k8s.io/kubectl v0.30.0/go.mod h1:zgolRw2MQXLPwmic2l/+iHs239L49fhSeICuMhQQXTI=
+k8s.io/kubectl v0.30.3 h1:YIBBvMdTW0xcDpmrOBzcpUVsn+zOgjMYIu7kAq+yqiI=
+k8s.io/kubectl v0.30.3/go.mod h1:IcR0I9RN2+zzTRUa1BzZCm4oM0NLOawE6RzlDvd1Fpo=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=

helm/kube_resources.go

@@ -0,0 +1,224 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package helm
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/kube"
+	"helm.sh/helm/v3/pkg/release"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/managedfields"
+	"k8s.io/cli-runtime/pkg/genericiooptions"
+	"k8s.io/cli-runtime/pkg/resource"
+	"k8s.io/client-go/discovery"
+	"k8s.io/kube-openapi/pkg/util/proto"
+	"k8s.io/kubectl/pkg/cmd/diff"
+	"k8s.io/kubectl/pkg/scheme"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+)
+
+func getKubeClient(actionConfig *action.Configuration) (*kube.Client, error) {
+	kc, ok := actionConfig.KubeClient.(*kube.Client)
+	if !ok {
+		return nil, errors.Errorf("client is not a *kube.Client")
+	}
+	return kc, nil
+}
+
+// regenerateGVKParser builds the parser from the raw OpenAPI schema.
+func regenerateGVKParser(dc discovery.DiscoveryInterface) (*managedfields.GvkParser, error) {
+	doc, err := dc.OpenAPISchema()
+	if err != nil {
+		return nil, err
+	}
+
+	models, err := proto.NewOpenAPIData(doc)
+	if err != nil {
+		return nil, err
+	}
+
+	return managedfields.NewGVKParser(models, false)
+}
+
+// removeUnmanagedFields removes fields updated by `kube-controller-manager` or
+// through subresource apis from a kubernetes object
+func removeUnmanagedFields(parser *managedfields.GvkParser, obj runtime.Object, gvk schema.GroupVersionKind) error {
+	parseableType := parser.Type(gvk)
+	if parseableType == nil {
+		return errors.Errorf("no parseable type found for %s", gvk.String())
+	}
+	typedObj, err := parseableType.FromStructured(obj)
+	if err != nil {
+		return err
+	}
+	accessor, err := apimeta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+	objManagedFields := accessor.GetManagedFields()
+	fieldSet := &fieldpath.Set{}
+	for _, mf := range objManagedFields {
+		if mf.Manager == "kube-controller-manager" || mf.Subresource != "" {
+			fs := &fieldpath.Set{}
+			if err := fs.FromJSON(bytes.NewReader(mf.FieldsV1.Raw)); err != nil {
+				return err
+			}
+			fieldSet = fieldSet.Union(fs)
+		}
+	}
+	u := typedObj.RemoveItems(fieldSet).AsValue().Unstructured()
+	m, ok := u.(map[string]interface{})
+	if !ok {
+		return errors.Errorf("unexpected type %T", u)
+	}
+	return runtime.DefaultUnstructuredConverter.FromUnstructured(m, obj)
+}
+
+// mapRuntimeObjects maps a list of kubernetes objects by key to their JSON
+// representation, with unmanaged fields removed and sensitive values redacted
+func mapRuntimeObjects(kc *kube.Client, objects []runtime.Object, d resourceGetter) (map[string]string, error) {
+	clientSet, err := kc.Factory.KubernetesClientSet()
+	if err != nil {
+		return nil, err
+	}
+	parser, err := regenerateGVKParser(clientSet.Discovery())
+	if err != nil {
+		return nil, err
+	}
+
+	mappedObjects := make(map[string]string)
+	for _, obj := range objects {
+		gvk := obj.GetObjectKind().GroupVersionKind()
+		if gvk.Kind == "Secret" {
+			secret := &corev1.Secret{}
+			err := scheme.Scheme.Convert(obj, secret, nil)
+			if err != nil {
+				return nil, err
+			}
+			redactSecretData(secret)
+			obj = secret
+		}
+		accessor, err := apimeta.Accessor(obj)
+		if err != nil {
+			return nil, err
+		}
+		key := fmt.Sprintf("%s/%s/%s/%s",
+			strings.ToLower(gvk.GroupKind().String()),
+			gvk.Version,
+			accessor.GetNamespace(),
+			accessor.GetName(),
+		)
+		if err := removeUnmanagedFields(parser, obj, gvk); err != nil {
+			return nil, err
+		}
+		accessor.SetUID(types.UID(""))
+		accessor.SetCreationTimestamp(metav1.Time{})
+		accessor.SetResourceVersion("")
+		accessor.SetManagedFields(nil)
+		objJSON, err := json.Marshal(obj)
+		if err != nil {
+			return nil, err
+		}
+		mappedObjects[key] = redactSensitiveValues(string(objJSON), d)
+	}
+	return mappedObjects, nil
+}
+
+func mapResources(actionConfig *action.Configuration, r *release.Release, d resourceGetter, f func(*resource.Info) (runtime.Object, error)) (map[string]string, error) {
+	resources, err := actionConfig.KubeClient.Build(bytes.NewBufferString(r.Manifest), false)
+	if err != nil {
+		return nil, err
+	}
+	var objects []runtime.Object
+	err = resources.Visit(func(i *resource.Info, err error) error {
+		if err != nil {
+			return err
+		}
+		obj, err := f(i)
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		objects = append(objects, obj)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	kc, err := getKubeClient(actionConfig)
+	if err != nil {
+		return nil, err
+	}
+	return mapRuntimeObjects(kc, objects, d)
+}
+
+// getLiveResources gets the live kubernetes resources of a release
+func getLiveResources(r *release.Release, m *Meta, d resourceGetter) (map[string]string, error) {
+	actionConfig, err := m.GetHelmConfiguration(r.Namespace)
+	if err != nil {
+		return nil, err
+	}
+	kc, err := getKubeClient(actionConfig)
+	if err != nil {
+		return nil, err
+	}
+	return mapResources(actionConfig, r, d, func(i *resource.Info) (runtime.Object, error) {
+		gvk := i.Object.GetObjectKind().GroupVersionKind()
+		return kc.Factory.NewBuilder().
+			Unstructured().
+			NamespaceParam(i.Namespace).DefaultNamespace().
+			ResourceNames(gvk.GroupKind().String(), i.Name).
+			Flatten().
+			Do().
+			Object()
+	})
+}
+
+// getDryRunResources gets the kubernetes resources as they would look like if
+// the helm manifest is applied to the cluster. this is useful for detecting the
+// differences between the live cluster state and the desired state.
+func getDryRunResources(r *release.Release, m *Meta, d resourceGetter) (map[string]string, error) {
+	actionConfig, err := m.GetHelmConfiguration(r.Namespace)
+	if err != nil {
+		return nil, err
+	}
+	ioStreams := genericiooptions.IOStreams{
+		In:     os.Stdin,
+		Out:    os.Stdout,
+		ErrOut: os.Stderr,
+	}
+	fieldManager := "terraform-provider-helm"
+	if os.Args[0] != "" {
+		fieldManager = filepath.Base(os.Args[0])
+	}
+	return mapResources(actionConfig, r, d, func(i *resource.Info) (runtime.Object, error) {
+		info := &diff.InfoObject{
+			LocalObj:        i.Object,
+			Info:            i,
+			Encoder:         scheme.DefaultJSONEncoder(),
+			Force:           false,
+			ServerSideApply: true,
+			FieldManager:    fieldManager,
+			ForceConflicts:  true,
+			IOStreams:       ioStreams,
+		}
+		return info.Merged()
+	})
+}

helm/manifest_json.go

@@ -56,10 +56,7 @@ func convertYAMLManifestToJSON(manifest string) (string, error) {
 				return "", err
 			}
 
-			for k, v := range secret.Data {
-				h := hashSensitiveValue(string(v))
-				secret.Data[k] = []byte(h)
-			}
+			redactSecretData(&secret)
 
 			jsonbytes, err = json.Marshal(secret)
 			if err != nil {
@@ -89,6 +86,14 @@ func hashSensitiveValue(v string) string {
 	return fmt.Sprintf("(sensitive value %x)", hash)
 }
 
+// redactSecretData redacts the `Data` of a Secret
+func redactSecretData(secret *corev1.Secret) {
+	for k, v := range secret.Data {
+		h := hashSensitiveValue(string(v))
+		secret.Data[k] = []byte(h)
+	}
+}
+
 // redactSensitiveValues removes values that appear in `set_sensitive` blocks from
 // the manifest JSON
 func redactSensitiveValues(text string, d resourceGetter) string {