feat: add pod-level minimum/maximum bounds

README.md

@@ -14,14 +14,16 @@ Helps you resize pods created by a DaemonSet depending on the amount of allocata
     - `node-specific-sizing.manomano.tech/request-memory-fraction: 0.1`
     - `node-specific-sizing.manomano.tech/limit-memory-fraction: 0.1`
 
-3. Optionally set absolute minimums, maximums and exclusions
-   NB: Only pods with the `node-specific-sizing.manomano.tech/enabled: "true"` label will see their resource modified.
-   - `node-specific-sizing.manomano.tech/minimum-cpu-request: 0.5`
-   - `node-specific-sizing.manomano.tech/minimum-cpu-limit: 0.5`
-   - `node-specific-sizing.manomano.tech/maximum-memory-request: 0.5`
-   - `node-specific-sizing.manomano.tech/maximum-memory-limit: 0.5`
-
-4. Optionally exclude some containers from dynamic-sizing
+3. *Optionally*, set up appropriate minimums and maximums.
+   - `node-specific-sizing.manomano.tech/minimum-cpu: 50m`
+   - `node-specific-sizing.manomano.tech/maximum-cpu: 4`
+   - `node-specific-sizing.manomano.tech/minimum-memory: 50M`
+   - `node-specific-sizing.manomano.tech/maximum-memory: 4G`
+   - NOTE: Minimums and maximums are applied to both resource and limits. 
+     We don't see the need to add different minimums for requests in limits in practice. You may challenge that choice by opening an issue.
+   - NOTE: Minimums and maximums are to be understood per-pod and not per-container. See resource-sizing algorithm for details.
+
+4. *Optionally*, exclude some containers from dynamic sizing.
     - `node-specific-sizing.manomano.tech/exclude-containers: istio-init,istio-proxy`
     - NOT IMPLEMENTED
 
@@ -43,7 +45,7 @@ follows:
   For any given container, `relative_tunable = container_tunable / (sum(container_tunables) - sum(excluded_container_tunables))` 
 - Derive a `pod_tunable_budget = allocatable_tunable_on_node * configured_pod_proportion - sum(excluded_container_tunables)`. This represents the resources that will be given to the pod.
 - Clamp `pod_tunable_budget` if minimums and/or maximums are set for that tunable.
-- Finally, `new_absolute_tunable = pod_tunable_budget * relative_tunable` spreads the budget around.
+- Finally, `new_absolute_tunable = pod_tunable_budget * relative_tunable` spreads the budget between containers.
 
 Exclusions and clamping notwithstanding, the requests/limits proportions between the different containers do not vary with node specific sizing.
 

bin/playground.sh

@@ -100,6 +100,8 @@ if [[ "$OPT_IN_MODE" == 0 || "$WITH_WORKLOAD" == 1 ]]; then
         annotations:
           node-specific-sizing.manomano.tech/request-memory-fraction: "0.05"
           node-specific-sizing.manomano.tech/limit-memory-fraction: "0.09"
+          node-specific-sizing.manomano.tech/minimum-memory: "40M"
+          node-specific-sizing.manomano.tech/maximum-memory: "742M"
       spec:
         terminationGracePeriodSeconds: 0
         containers:

bin/print_resources.sh

@@ -2,5 +2,24 @@
 
 set -eo pipefail
 
-kubectl -n default get -o json pods | \
-    jq -r '.items[] | .status.hostIP as $hip | .spec.containers[] | .name as $n | .resources | "\($hip) - \($n) - REQ \(.requests.memory) - LIM \(.limits.memory)"'
+_pod_json=$(kubectl -n default get -o json pods)
+_len=$(echo "$_pod_json" | jq -r '.items | length - 1')
+
+# jq -r '.spec.nodeName as $hip | .spec.containers[] | .name as $n | .resources | "\($hip) - \($n) - REQ \(.requests.memory) - LIM \(.limits.memory)"'
+
+for i in `seq 0 "$_len"`; do
+  _node=$(echo "$_pod_json" | jq -r ".items[$i] | .spec.nodeName")
+  _containers_count=$(echo "$_pod_json" | jq ".items[$i] | .spec.containers | length - 1")
+
+  for j in `seq 0 "$_containers_count"`; do
+    _name=$(echo "$_pod_json" | jq -r ".items[$i] | .spec.containers[$j] | .name")
+    _cpu_req=$(echo "$_pod_json" | jq -r ".items[$i] | .spec.containers[$j] | .resources.requests.cpu")
+    _mem_req=$(echo "$_pod_json" | jq -r ".items[$i] | .spec.containers[$j] | .resources.requests.memory")
+    _cpu_lim=$(echo "$_pod_json" | jq -r ".items[$i] | .spec.containers[$j] | .resources.limits.cpu")
+    _mem_lim=$(echo "$_pod_json" | jq -r ".items[$i] | .spec.containers[$j] | .resources.limits.memory")
+
+    # shellcheck disable=SC2182
+    printf "%-20s cpu: req=%-5s lim=%-5s\n" "$_node" "$_cpu_req" "$_cpu_lim"
+    printf "%-20s mem: req=%-5s lim=%-5s\n" "--- $_name" "$_mem_req" "$_mem_lim"
+  done
+done

cmd/pod_patcher.go

@@ -34,14 +34,15 @@ func computeProportionalResourceRequirements(pod *corev1.Pod) map[string]*rps.Re
 	return containerRequirements
 }
 
-func computePodResourceBudget(fractions *rps.ResourceProperties, node *corev1.Node) *rps.ResourceProperties {
+func computePodResourceBudget(userSettings *rps.ResourceProperties, node *corev1.Node) *rps.ResourceProperties {
 	podResourceBudget := rps.New()
-	for prop := range fractions.All() {
+	for prop := range userSettings.All() {
 		if nodeCapacity, ok := node.Status.Capacity[prop.ResourceName()]; ok {
 			qty := nodeCapacity.AsApproximateFloat64()
-			podResourceBudget.BindPropertyFloat(prop.Property(), prop.ResourceName(), qty*prop.Value())
+			podResourceBudget.BindPropertyFloat(rps.ResourceQuantity, prop.Property(), prop.ResourceName(), qty*prop.Value())
 		}
 	}
+	podResourceBudget.ClampRequestsAndLimits(userSettings)
 	return podResourceBudget
 }
 
@@ -61,7 +62,10 @@ func multiplyQuantity(quantity resource.Quantity, multiplier float64) *resource.
 	}
 }
 
-func computePodContainerResourceBudget(containersProportionalResourceRequirements map[string]*rps.ResourceProperties, podResourceBudget *rps.ResourceProperties) map[string]*rps.ResourceProperties {
+func computePodContainerResourceBudget(
+	containersProportionalResourceRequirements map[string]*rps.ResourceProperties,
+	podResourceBudget *rps.ResourceProperties,
+) map[string]*rps.ResourceProperties {
 	result := make(map[string]*rps.ResourceProperties)
 	for containerName, proportionalResourceRequirements := range containersProportionalResourceRequirements {
 		result[containerName] = proportionalResourceRequirements.Mul(podResourceBudget)
@@ -120,7 +124,7 @@ func createPatch(ctx context.Context, pod *corev1.Pod) ([]byte, error) {
 
 	zap.L().Debug("Starting patch process")
 
-	err, fractions := rps.NewFromAnnotations(pod.Annotations)
+	err, userSettings := rps.NewFromAnnotations(pod.Annotations)
 	if err != nil {
 		return nil, fmt.Errorf("problem parsing annotations: %w", err)
 	}
@@ -151,7 +155,7 @@ func createPatch(ctx context.Context, pod *corev1.Pod) ([]byte, error) {
 	// We need pod budget = node resources * nssConfig.nodeResourcesFractions
 	// When we have pod budget we want pod container budget = podBudget * containersProportionalRequirements
 	// Then set values
-	podResourceBudget := computePodResourceBudget(fractions, &node)
+	podResourceBudget := computePodResourceBudget(userSettings, &node)
 
 	zap.L().Debug("podResourceBudget", zap.Any("pRB", *podResourceBudget))
 

go.mod

@@ -1,33 +1,33 @@
 module github.com/ManoManoTech/kubernetes-node-specific-sizing
 
-go 1.23
+go 1.23.0
 
 require (
+	github.com/deckarep/golang-set/v2 v2.6.0
+	github.com/onsi/ginkgo/v2 v2.19.0
+	github.com/onsi/gomega v1.33.1
 	go.uber.org/zap v1.27.0
 	k8s.io/api v0.31.0
 	k8s.io/apimachinery v0.31.0
 	sigs.k8s.io/controller-runtime v0.19.0
 )
 
 require (
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -37,10 +37,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
@@ -51,12 +47,11 @@ require (
 	golang.org/x/term v0.23.0 // indirect
 	golang.org/x/text v0.17.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
-	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.31.0 // indirect
 	k8s.io/client-go v0.31.0 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 // indirect

go.sum

@@ -10,8 +10,6 @@ github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80N
 github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
 github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
-github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -102,8 +100,6 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
-go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=