fix: Remove the helm created internal load balancer and replace it wi…

…th an internal NLB that fronts the existing load balancer
wandb · Dec 20, 2024 · cedf4fb · cedf4fb
1 parent 30832bd
commit cedf4fb
Show file tree

Hide file tree

Showing 8 changed files with 92 additions and 82 deletions.
diff --git a/main.tf b/main.tf
@@ -129,10 +129,12 @@ module "app_gke" {
 }
 
 module "cloud_nat" {
-  count     = var.enable_private_gke_nodes ? 1 : 0
+  count     = var.enable_private_gke_nodes || var.create_private_link ? 1 : 0
   source    = "./modules/cloud_nat"
   network   = local.network
   namespace = var.namespace
+  vpc_nat   = var.enable_private_gke_nodes
+  proxy_nat = var.create_private_link
 }
 
 module "app_lb" {
@@ -254,7 +256,6 @@ locals {
     "OIDC_AUTH_METHOD" = var.oidc_auth_method
     "OIDC_SECRET"      = var.oidc_secret
   } : {}
-  internal_lb_name = "${var.namespace}-internal"
 }
 
 locals {
@@ -343,15 +344,6 @@ module "wandb" {
           "kubernetes.io/ingress.global-static-ip-name" = module.app_lb.address_operator_name
           "ingress.gcp.kubernetes.io/pre-shared-cert"   = module.app_lb.certificate
         }
-        ## In order to support secondary ingress required min version 0.13.0 of operator-wandb chart
-        secondary = {
-          create       = var.create_private_link # internal ingress for private link connections
-          nameOverride = local.internal_lb_name
-          annotations = {
-            "kubernetes.io/ingress.class"                   = "gce-internal"
-            "kubernetes.io/ingress.regional-static-ip-name" = var.create_private_link ? google_compute_address.default[0].name : null
-          }
-        }
       }
 
       # To support otel rds and redis metrics need operator-wandb chart minimum version 0.13.8 ( stackdriver subchart)
@@ -427,62 +419,16 @@ module "wandb" {
   ]
 }
 
-# proxy-only subnet used by internal load balancer
-resource "google_compute_subnetwork" "proxy" {
-  count         = var.create_private_link ? 1 : 0
-  name          = "${var.namespace}-proxy-subnet"
-  ip_cidr_range = var.ilb_proxynetwork_cidr
-  purpose       = "REGIONAL_MANAGED_PROXY"
-  role          = "ACTIVE"
-  network       = local.network.id
-  timeouts {
-    delete = "2m"
-  }
-}
-
-resource "null_resource" "wait_for_lb" {
-  count                 = var.create_private_link ? 1 : 0
-
-  provisioner "local-exec" {
-    command = <<EOF
-      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/$(echo $(uname -s) | tr '[:upper:]' '[:lower:]')/$(uname -m)/kubectl" && chmod +x kubectl &&
-      echo ${module.app_gke.cluster_ca_certificate} | base64 -d  >> kubectl_ca.crt &&
-      ./kubectl config set-cluster ${module.app_gke.cluster_name} --server=https://${module.app_gke.cluster_endpoint} --certificate-authority=./kubectl_ca.crt &&
-      ./kubectl config set-credentials ${module.app_gke.cluster_name} --token=${data.google_client_config.current.access_token} &&
-      ./kubectl config set-context ${module.app_gke.cluster_name} --cluster=${module.app_gke.cluster_name} --user=${module.app_gke.cluster_name} &&
-      ./kubectl config use-context ${module.app_gke.cluster_name} &&
-      ./kubectl wait --for=jsonpath='{.status.loadBalancer.ingress}' ingress --namespace="default" ${local.internal_lb_name}
-    EOF
-  }
-
-  depends_on = [module.wandb]
-}
-
-data "kubernetes_ingress_v1" "internal-lb" {
-  count       = var.create_private_link ? 1 : 0
-  metadata {
-    name      = local.internal_lb_name
-    namespace = "default"
-  }
-
-  depends_on = [null_resource.wait_for_lb]
-}
-
-locals {
-  annotations = try(data.kubernetes_ingress_v1.internal-lb[0].metadata[0].annotations, {})
-  forwarding_rule = try(local.annotations["ingress.kubernetes.io/forwarding-rule"], "")
-}
-
 ## In order to support private link required min version 0.13.0 of operator-wandb chart
 module "private_link" {
   count                 = var.create_private_link ? 1 : 0
   source                = "./modules/private_link"
   namespace             = var.namespace
-  forwarding_rule       = local.forwarding_rule
   network               = local.network
   subnetwork            = local.subnetwork
   allowed_project_names = var.allowed_project_names
   psc_subnetwork        = var.psc_subnetwork_cidr
   proxynetwork_cidr     = var.ilb_proxynetwork_cidr
-  depends_on            = [google_compute_subnetwork.proxy, null_resource.wait_for_lb]
+  fqdn                  = local.fqdn
+  depends_on            = [module.wandb]
 }
diff --git a/modules/cloud_nat/main.tf b/modules/cloud_nat/main.tf
@@ -14,10 +14,22 @@ resource "google_compute_address" "this" {
 
 # create cloud nat public gateway
 resource "google_compute_router_nat" "nat" {
+  count                              = var.vpc_nat ? 1 : 0
   name                               = "${var.namespace}-cloud-nat"
   router                             = google_compute_router.this.name
   region                             = google_compute_router.this.region
   nat_ip_allocate_option             = "MANUAL_ONLY"
   nat_ips                            = google_compute_address.this.*.self_link
   source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+}
+
+# create cloud nat public gateway for Private Service Connect LB
+resource "google_compute_router_nat" "nat_lb_proxy" {
+  count                              = var.proxy_nat ? 1 : 0
+  name                               = "${var.namespace}-cloud-nat-lb-proxy"
+  router                             = google_compute_router.this.name
+  region                             = google_compute_router.this.region
+  nat_ip_allocate_option             = "AUTO_ONLY"
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+  endpoint_types = ["ENDPOINT_TYPE_MANAGED_PROXY_LB"]
 }
diff --git a/modules/cloud_nat/variables.tf b/modules/cloud_nat/variables.tf
@@ -5,4 +5,14 @@ variable "namespace" {
 variable "network" {
   description = "Google Compute Engine network to which the cluster is connected."
   type        = object({ self_link = string })
+}
+
+variable "proxy_nat" {
+  description = "Enable NAT for the Load Balancer Proxy Subnets"
+  type = bool
+}
+
+variable "vpc_nat" {
+  description = "Enable NAT for the VPC"
+  type = bool
 }
diff --git a/modules/private_link/main.tf b/modules/private_link/main.tf
@@ -1,23 +1,67 @@
-terraform {
-  required_providers {
-    google = {
-      source  = "hashicorp/google"
-      version = "5.34.0"
-    }
-    null = {
-      source  = "hashicorp/null"
-      version = "3.2.2"
-    }
+data "google_client_config" "current" {}
+
+# proxy-only subnet used by internal load balancer
+resource "google_compute_subnetwork" "proxy" {
+  name          = "${var.namespace}-proxy-subnet"
+  ip_cidr_range = var.proxynetwork_cidr
+  purpose       = "REGIONAL_MANAGED_PROXY"
+  role          = "ACTIVE"
+  network       = var.network.id
+  timeouts {
+    delete = "2m"
   }
 }
-data "google_client_config" "current" {}
+
+resource "google_compute_region_network_endpoint_group" "external_lb" {
+  name = "${var.namespace}-psc-lb-neg"
+  region = data.google_client_config.current.region
+
+  network_endpoint_type = "INTERNET_FQDN_PORT"
+  network = var.network.id
+}
+
+resource "google_compute_region_network_endpoint" "external_lb" {
+  region_network_endpoint_group = google_compute_region_network_endpoint_group.external_lb.name
+
+  fqdn  = var.fqdn
+  port  = 443
+}
+
+resource "google_compute_region_backend_service" "internal_nlb" {
+  name                  = "${var.namespace}-psc-nlb"
+  protocol              = "TCP"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+  backend {
+    group           = google_compute_region_network_endpoint_group.external_lb.id
+    balancing_mode  = ""
+  }
+}
+
+resource "google_compute_region_target_tcp_proxy" "internal_nlb" {
+  name            = "${var.namespace}-psc-nlb"
+  backend_service = google_compute_region_backend_service.internal_nlb.id
+}
+
+resource "google_compute_forwarding_rule" "internal_nlb" {
+  name = "${var.namespace}-psc-nlb"
+  load_balancing_scheme = "INTERNAL_MANAGED"
+
+  allow_global_access = true
+  ip_protocol = "TCP"
+  port_range  = "443"
+
+  target = google_compute_region_target_tcp_proxy.internal_nlb.id
+
+  network = var.network.id
+  subnetwork = var.subnetwork.self_link
+}
 
 resource "google_compute_service_attachment" "default" {
   name                  = "${var.namespace}-private-link"
-  enable_proxy_protocol = false
   connection_preference = "ACCEPT_MANUAL"
+  enable_proxy_protocol = false
   nat_subnets           = [google_compute_subnetwork.default.id]
-  target_service        = "https://www.googleapis.com/compute/v1/projects/${data.google_client_config.current.project}/regions/${data.google_client_config.current.region}/forwardingRules/${var.forwarding_rule}"
+  target_service        = google_compute_forwarding_rule.internal_nlb.self_link
 
   dynamic "consumer_accept_lists" {
     for_each = var.allowed_project_names != {} ? var.allowed_project_names : {}

diff --git a/modules/private_link/variables.tf b/modules/private_link/variables.tf
@@ -3,12 +3,6 @@ variable "namespace" {
   description = "The name prefix for all resources created."
 }
 
-variable "labels" {
-  description = "Labels which will be applied to all applicable resources."
-  type        = map(string)
-  default     = {}
-}
-
 variable "network" {
   description = "Google Compute Engine network to which the cluster is connected."
   type        = object({ id = string })
@@ -37,7 +31,7 @@ variable "proxynetwork_cidr" {
   description = "Internal load balancer proxy subnetwork"
 }
 
-variable "forwarding_rule" {
+variable "fqdn" {
   type        = string
-  description = "forwarding rule name used in private service connect as a target"
+  description = "Fully qualified domain name or hostname"
 }
diff --git a/outputs.tf b/outputs.tf
@@ -92,7 +92,7 @@ output "database_instance_type" {
   value = local.database_machine_type
 }
 
-output "private_attachement_id" {
+output "private_attachment_id" {
   value = var.create_private_link ? module.private_link[0].private_attachement_id : null
 }
 

diff --git a/variables.tf b/variables.tf
@@ -359,7 +359,7 @@ variable "public_access" {
 variable "allowed_project_names" {
   type = map(number)
   default = {
-    # "project_ID" = 4 
+    "wandb-qa" : 20
   }
   description = "A map of allowed projects where each key is a project number and the value is the connection limit."
 }

diff --git a/versions.tf b/versions.tf
@@ -9,6 +9,10 @@ terraform {
       source  = "hashicorp/google"
       version = "~> 5.30"
     }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 5.30"
+    }
     kubernetes = {
       source  = "hashicorp/kubernetes"
       version = "~> 2.23"