diff --git a/main.tf b/main.tf index 49989db..957d7a7 100644 --- a/main.tf +++ b/main.tf @@ -129,10 +129,12 @@ module "app_gke" { } module "cloud_nat" { - count = var.enable_private_gke_nodes ? 1 : 0 + count = var.enable_private_gke_nodes || var.create_private_link ? 1 : 0 source = "./modules/cloud_nat" network = local.network namespace = var.namespace + vpc_nat = var.enable_private_gke_nodes + proxy_nat = var.create_private_link } module "app_lb" { @@ -254,7 +256,6 @@ locals { "OIDC_AUTH_METHOD" = var.oidc_auth_method "OIDC_SECRET" = var.oidc_secret } : {} - internal_lb_name = "${var.namespace}-internal" } locals { @@ -343,15 +344,6 @@ module "wandb" { "kubernetes.io/ingress.global-static-ip-name" = module.app_lb.address_operator_name "ingress.gcp.kubernetes.io/pre-shared-cert" = module.app_lb.certificate } - ## In order to support secondary ingress required min version 0.13.0 of operator-wandb chart - secondary = { - create = var.create_private_link # internal ingress for private link connections - nameOverride = local.internal_lb_name - annotations = { - "kubernetes.io/ingress.class" = "gce-internal" - "kubernetes.io/ingress.regional-static-ip-name" = var.create_private_link ? google_compute_address.default[0].name : null - } - } } # To support otel rds and redis metrics need operator-wandb chart minimum version 0.13.8 ( stackdriver subchart) @@ -427,62 +419,16 @@ module "wandb" { ] } -# proxy-only subnet used by internal load balancer -resource "google_compute_subnetwork" "proxy" { - count = var.create_private_link ? 1 : 0 - name = "${var.namespace}-proxy-subnet" - ip_cidr_range = var.ilb_proxynetwork_cidr - purpose = "REGIONAL_MANAGED_PROXY" - role = "ACTIVE" - network = local.network.id - timeouts { - delete = "2m" - } -} - -resource "null_resource" "wait_for_lb" { - count = var.create_private_link ? 1 : 0 - - provisioner "local-exec" { - command = <> kubectl_ca.crt && - ./kubectl config set-cluster ${module.app_gke.cluster_name} --server=https://${module.app_gke.cluster_endpoint} --certificate-authority=./kubectl_ca.crt && - ./kubectl config set-credentials ${module.app_gke.cluster_name} --token=${data.google_client_config.current.access_token} && - ./kubectl config set-context ${module.app_gke.cluster_name} --cluster=${module.app_gke.cluster_name} --user=${module.app_gke.cluster_name} && - ./kubectl config use-context ${module.app_gke.cluster_name} && - ./kubectl wait --for=jsonpath='{.status.loadBalancer.ingress}' ingress --namespace="default" ${local.internal_lb_name} - EOF - } - - depends_on = [module.wandb] -} - -data "kubernetes_ingress_v1" "internal-lb" { - count = var.create_private_link ? 1 : 0 - metadata { - name = local.internal_lb_name - namespace = "default" - } - - depends_on = [null_resource.wait_for_lb] -} - -locals { - annotations = try(data.kubernetes_ingress_v1.internal-lb[0].metadata[0].annotations, {}) - forwarding_rule = try(local.annotations["ingress.kubernetes.io/forwarding-rule"], "") -} - ## In order to support private link required min version 0.13.0 of operator-wandb chart module "private_link" { count = var.create_private_link ? 1 : 0 source = "./modules/private_link" namespace = var.namespace - forwarding_rule = local.forwarding_rule network = local.network subnetwork = local.subnetwork allowed_project_names = var.allowed_project_names psc_subnetwork = var.psc_subnetwork_cidr proxynetwork_cidr = var.ilb_proxynetwork_cidr - depends_on = [google_compute_subnetwork.proxy, null_resource.wait_for_lb] + fqdn = local.fqdn + depends_on = [module.wandb] } diff --git a/modules/cloud_nat/main.tf b/modules/cloud_nat/main.tf index 5e6f885..dbf77ba 100644 --- a/modules/cloud_nat/main.tf +++ b/modules/cloud_nat/main.tf @@ -14,10 +14,22 @@ resource "google_compute_address" "this" { # create cloud nat public gateway resource "google_compute_router_nat" "nat" { + count = var.vpc_nat ? 1 : 0 name = "${var.namespace}-cloud-nat" router = google_compute_router.this.name region = google_compute_router.this.region nat_ip_allocate_option = "MANUAL_ONLY" nat_ips = google_compute_address.this.*.self_link source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" +} + +# create cloud nat public gateway for Private Service Connect LB +resource "google_compute_router_nat" "nat_lb_proxy" { + count = var.proxy_nat ? 1 : 0 + name = "${var.namespace}-cloud-nat-lb-proxy" + router = google_compute_router.this.name + region = google_compute_router.this.region + nat_ip_allocate_option = "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" + endpoint_types = ["ENDPOINT_TYPE_MANAGED_PROXY_LB"] } \ No newline at end of file diff --git a/modules/cloud_nat/variables.tf b/modules/cloud_nat/variables.tf index 3fc8204..4e08df8 100644 --- a/modules/cloud_nat/variables.tf +++ b/modules/cloud_nat/variables.tf @@ -5,4 +5,14 @@ variable "namespace" { variable "network" { description = "Google Compute Engine network to which the cluster is connected." type = object({ self_link = string }) +} + +variable "proxy_nat" { + description = "Enable NAT for the Load Balancer Proxy Subnets" + type = bool +} + +variable "vpc_nat" { + description = "Enable NAT for the VPC" + type = bool } \ No newline at end of file diff --git a/modules/private_link/main.tf b/modules/private_link/main.tf index 0946680..43450c4 100644 --- a/modules/private_link/main.tf +++ b/modules/private_link/main.tf @@ -1,23 +1,67 @@ -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = "5.34.0" - } - null = { - source = "hashicorp/null" - version = "3.2.2" - } +data "google_client_config" "current" {} + +# proxy-only subnet used by internal load balancer +resource "google_compute_subnetwork" "proxy" { + name = "${var.namespace}-proxy-subnet" + ip_cidr_range = var.proxynetwork_cidr + purpose = "REGIONAL_MANAGED_PROXY" + role = "ACTIVE" + network = var.network.id + timeouts { + delete = "2m" } } -data "google_client_config" "current" {} + +resource "google_compute_region_network_endpoint_group" "external_lb" { + name = "${var.namespace}-psc-lb-neg" + region = data.google_client_config.current.region + + network_endpoint_type = "INTERNET_FQDN_PORT" + network = var.network.id +} + +resource "google_compute_region_network_endpoint" "external_lb" { + region_network_endpoint_group = google_compute_region_network_endpoint_group.external_lb.name + + fqdn = var.fqdn + port = 443 +} + +resource "google_compute_region_backend_service" "internal_nlb" { + name = "${var.namespace}-psc-nlb" + protocol = "TCP" + load_balancing_scheme = "INTERNAL_MANAGED" + backend { + group = google_compute_region_network_endpoint_group.external_lb.id + balancing_mode = "" + } +} + +resource "google_compute_region_target_tcp_proxy" "internal_nlb" { + name = "${var.namespace}-psc-nlb" + backend_service = google_compute_region_backend_service.internal_nlb.id +} + +resource "google_compute_forwarding_rule" "internal_nlb" { + name = "${var.namespace}-psc-nlb" + load_balancing_scheme = "INTERNAL_MANAGED" + + allow_global_access = true + ip_protocol = "TCP" + port_range = "443" + + target = google_compute_region_target_tcp_proxy.internal_nlb.id + + network = var.network.id + subnetwork = var.subnetwork.self_link +} resource "google_compute_service_attachment" "default" { name = "${var.namespace}-private-link" - enable_proxy_protocol = false connection_preference = "ACCEPT_MANUAL" + enable_proxy_protocol = false nat_subnets = [google_compute_subnetwork.default.id] - target_service = "https://www.googleapis.com/compute/v1/projects/${data.google_client_config.current.project}/regions/${data.google_client_config.current.region}/forwardingRules/${var.forwarding_rule}" + target_service = google_compute_forwarding_rule.internal_nlb.self_link dynamic "consumer_accept_lists" { for_each = var.allowed_project_names != {} ? var.allowed_project_names : {} diff --git a/modules/private_link/variables.tf b/modules/private_link/variables.tf index ecedb1e..182926b 100644 --- a/modules/private_link/variables.tf +++ b/modules/private_link/variables.tf @@ -3,12 +3,6 @@ variable "namespace" { description = "The name prefix for all resources created." } -variable "labels" { - description = "Labels which will be applied to all applicable resources." - type = map(string) - default = {} -} - variable "network" { description = "Google Compute Engine network to which the cluster is connected." type = object({ id = string }) @@ -37,7 +31,7 @@ variable "proxynetwork_cidr" { description = "Internal load balancer proxy subnetwork" } -variable "forwarding_rule" { +variable "fqdn" { type = string - description = "forwarding rule name used in private service connect as a target" + description = "Fully qualified domain name or hostname" } \ No newline at end of file diff --git a/outputs.tf b/outputs.tf index f691c64..9a6377c 100644 --- a/outputs.tf +++ b/outputs.tf @@ -92,7 +92,7 @@ output "database_instance_type" { value = local.database_machine_type } -output "private_attachement_id" { +output "private_attachment_id" { value = var.create_private_link ? module.private_link[0].private_attachement_id : null } diff --git a/variables.tf b/variables.tf index 6627894..d9ff44f 100644 --- a/variables.tf +++ b/variables.tf @@ -359,7 +359,7 @@ variable "public_access" { variable "allowed_project_names" { type = map(number) default = { - # "project_ID" = 4 + "wandb-qa" : 20 } description = "A map of allowed projects where each key is a project number and the value is the connection limit." } diff --git a/versions.tf b/versions.tf index ca2a967..18d7e05 100644 --- a/versions.tf +++ b/versions.tf @@ -9,6 +9,10 @@ terraform { source = "hashicorp/google" version = "~> 5.30" } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 5.30" + } kubernetes = { source = "hashicorp/kubernetes" version = "~> 2.23"