chore(dev): Add system-reserved option for node pool creation (#176)

* chore(dev): Add system-reserved option for node pool creation

* Fixup! Add plumbing for top level vars

* Fixup! Set default values

examples/public-dns-external/main.tf

@@ -42,6 +42,11 @@ module "wandb_infra" {
   bucket_kms_key_arn = var.bucket_kms_key_arn
   use_internal_queue = true
   size               = var.size
+
+  system_reserved_cpu_millicores      = var.system_reserved_cpu_millicores
+  system_reserved_memory_megabytes    = var.system_reserved_memory_megabytes
+  system_reserved_ephemeral_megabytes = var.system_reserved_ephemeral_megabytes
+  system_reserved_pid                 = var.system_reserved_pid
 }
 
 data "aws_eks_cluster" "app_cluster" {
@@ -126,4 +131,4 @@ output "redis_instance_type" {
 
 output "standardized_size" {
   value = var.size
-}
+}

examples/public-dns-external/variables.tf

@@ -96,3 +96,27 @@ variable "other_wandb_env" {
   description = "Extra environment variables for W&B"
   default     = {}
 }
+
+variable "system_reserved_cpu_millicores" {
+  description = "(Optional) The amount of 'system-reserved' CPU millicores to pass to the kubelet. For example: 100.  A value of -1 disables the flag."
+  type        = number
+  default     = -1
+}
+
+variable "system_reserved_memory_megabytes" {
+  description = "(Optional) The amount of 'system-reserved' memory in megabytes to pass to the kubelet. For example: 100.  A value of -1 disables the flag."
+  type        = number
+  default     = -1
+}
+
+variable "system_reserved_ephemeral_megabytes" {
+  description = "(Optional) The amount of 'system-reserved' ephemeral storage in megabytes to pass to the kubelet. For example: 1000.  A value of -1 disables the flag."
+  type        = number
+  default     = -1
+}
+
+variable "system_reserved_pid" {
+  description = "(Optional) The amount of 'system-reserved' process ids [pid] to pass to the kubelet. For example: 1000.  A value of -1 disables the flag."
+  type        = number
+  default     = -1
+}

main.tf

@@ -143,6 +143,11 @@ module "app_eks" {
   cluster_endpoint_public_access_cidrs = var.kubernetes_public_access_cidrs
 
   eks_policy_arns = var.eks_policy_arns
+
+  system_reserved_cpu_millicores      = var.system_reserved_cpu_millicores
+  system_reserved_memory_megabytes    = var.system_reserved_memory_megabytes
+  system_reserved_ephemeral_megabytes = var.system_reserved_ephemeral_megabytes
+  system_reserved_pid                 = var.system_reserved_pid
 }
 
 module "app_lb" {

modules/app_eks/main.tf

@@ -4,6 +4,13 @@ locals {
   mysql_port         = 3306
   redis_port         = 6379
   encrypt_ebs_volume = true
+  system_reserved = join(",", flatten([
+    var.system_reserved_cpu_millicores >= 0 ? ["cpu=${var.system_reserved_cpu_millicores}m"] : [],
+    var.system_reserved_memory_megabytes >= 0 ? ["memory=${var.system_reserved_memory_megabytes}Mi"] : [],
+    var.system_reserved_ephemeral_megabytes >= 0 ? ["ephemeral-storage=${var.system_reserved_ephemeral_megabytes}Mi"] : [],
+    var.system_reserved_pid >= 0 ? ["pid=${var.system_reserved_pid}"] : []
+  ]))
+  create_launch_template = (local.encrypt_ebs_volume || local.system_reserved != "")
 }
 
 
@@ -67,7 +74,7 @@ module "eks" {
   node_groups = {
     primary = {
       # IMDsv2
-      create_launch_template               = local.encrypt_ebs_volume,
+      create_launch_template               = local.create_launch_template,
       desired_capacity                     = 2,
       disk_encrypted                       = local.encrypt_ebs_volume,
       disk_kms_key_id                      = var.kms_key_arn,
@@ -76,6 +83,7 @@ module "eks" {
       force_update_version                 = local.encrypt_ebs_volume,
       iam_role_arn                         = aws_iam_role.node.arn,
       instance_types                       = var.instance_types,
+      kubelet_extra_args                   = local.system_reserved != "" ? "--system-reserved=${local.system_reserved}" : "",
       max_capacity                         = 5,
       metadata_http_put_response_hop_limit = 2
       metadata_http_tokens                 = "required",

modules/app_eks/variables.tf

@@ -121,3 +121,27 @@ variable "desired_capacity" {
   type        = number
   default     = 2
 }
+
+variable "system_reserved_cpu_millicores" {
+  description = "(Optional) The amount of 'system-reserved' CPU millicores to pass to the kubelet. For example: 100.  A value of -1 disables the flag."
+  type        = number
+  default     = -1
+}
+
+variable "system_reserved_memory_megabytes" {
+  description = "(Optional) The amount of 'system-reserved' memory in megabytes to pass to the kubelet. For example: 100.  A value of -1 disables the flag."
+  type        = number
+  default     = -1
+}
+
+variable "system_reserved_ephemeral_megabytes" {
+  description = "(Optional) The amount of 'system-reserved' ephemeral storage in megabytes to pass to the kubelet. For example: 1000.  A value of -1 disables the flag."
+  type        = number
+  default     = -1
+}
+
+variable "system_reserved_pid" {
+  description = "(Optional) The amount of 'system-reserved' process ids [pid] to pass to the kubelet. For example: 1000.  A value of -1 disables the flag."
+  type        = number
+  default     = -1
+}

variables.tf

@@ -343,6 +343,30 @@ variable "eks_policy_arns" {
   default     = []
 }
 
+variable "system_reserved_cpu_millicores" {
+  description = "(Optional) The amount of 'system-reserved' CPU millicores to pass to the kubelet. For example: 100.  A value of -1 disables the flag."
+  type        = number
+  default     = 70
+}
+
+variable "system_reserved_memory_megabytes" {
+  description = "(Optional) The amount of 'system-reserved' memory in megabytes to pass to the kubelet. For example: 100.  A value of -1 disables the flag."
+  type        = number
+  default     = 100
+}
+
+variable "system_reserved_ephemeral_megabytes" {
+  description = "(Optional) The amount of 'system-reserved' ephemeral storage in megabytes to pass to the kubelet. For example: 1000.  A value of -1 disables the flag."
+  type        = number
+  default     = 750
+}
+
+variable "system_reserved_pid" {
+  description = "(Optional) The amount of 'system-reserved' process ids [pid] to pass to the kubelet. For example: 1000.  A value of -1 disables the flag."
+  type        = number
+  default     = 500
+}
+
 ##########################################
 # External Bucket                        #
 ##########################################