Skip to content

Commit

Permalink
Expose EPP and WHOIS endpoints on reginal load balancers
Browse files Browse the repository at this point in the history
k8s does not have a way to expose a global load balancer with TCP
endpoints, and setting up node port-based routing is a chore, even with
Terraform (which is what we did with the standalone proxy).

We will use Cloud DNS's geolocation routing policy to ensure that
clients connect to the endpoint closest to them.
  • Loading branch information
jianglai committed Dec 19, 2024
1 parent c9c61e4 commit 30137b5
Show file tree
Hide file tree
Showing 4 changed files with 215 additions and 12 deletions.
6 changes: 4 additions & 2 deletions jetty/deploy-nomulus-for-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,15 @@ do
sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
sed s/ENVIRONMENT/"${environment}"/g | \
sed s/PROXY_ENV/"${environment}"/g | \
sed s/PROXY_NAME/"proxy"/g | \
sed s/EPP/"epp"/g | \
sed s/WHOIS/"whois"/g | \
kubectl apply -f -
# canary
sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
sed s/ENVIRONMENT/"${environment}"/g | \
sed s/PROXY_ENV/"${environment}_canary"/g | \
sed s/PROXY_NAME/"proxy-canary"/g | \
sed s/EPP/"epp-canary"/g | \
sed s/WHOIS/"whois-canary"/g | \
sed s/"${service}"/"${service}-canary"/g | \
kubectl apply -f -
done
Expand Down
167 changes: 167 additions & 0 deletions jetty/get-endpoints.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
#! /bin/env python3
# Copyright 2024 The Nomulus Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''
A script that outputs the IP endpoints of various load balancers, to be run
after Nomulus is deployed.
'''

import ipaddress
import json
import subprocess
import sys
from dataclasses import dataclass
from ipaddress import IPv4Address
from ipaddress import IPv6Address
from operator import attrgetter
from operator import methodcaller


class PreserveContext:
def __enter__(self):
self._context = run_command('kubectl config current-context')

def __exit__(self, type, value, traceback):
run_command('kubectl config use-context ' + self._context)


class UseCluster(PreserveContext):
def __init__(self, cluster: str, region: str, project: str):
self._cluster = cluster
self._region = region
self._project = project

def __enter__(self):
super().__enter__()
cmd = f'gcloud container clusters get-credentials {self._cluster} --location {self._region} --project {self._project}'
run_command(cmd)


def run_command(cmd: str, print_output=False) -> str:
proc = subprocess.run(cmd, text=True, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if print_output:
print(proc.stdout)
return proc.stdout


def get_clusters(project: str) -> dict[str, str]:
cmd = f'gcloud container clusters list --project {project} --format=json'
content = json.loads(run_command(cmd))
res = {}
for item in content:
name = item['name']
region = item['location']
if not name.startswith('nomulus-cluster'):
continue
res[name] = region
return res


def get_endpoints(resource: str, service: str, selector: list[str]) -> list[
str]:
content = json.loads(
run_command(f'kubectl get {resource}/{service} -o json'))
name = content['metadata']['name']

Check notice

Code scanning / CodeQL

Unused local variable Note

Variable name is not used.
ips = content
for key in selector[:-1]:
ips = ips[key]
return [x[selector[-1]] for x in ips]


def get_region_symbol(region: str) -> str:
if region.startswith('us'):
return 'amer'
if region.startswith('europe'):
return 'emea'
if region.startswith('asia'):
return 'apac'
return 'other'


@dataclass
class IP:
service: str
region: str
address: IPv4Address | IPv6Address

def is_ipv6(self) -> bool:
return self.address.version == 6

def __str__(self) -> str:
return f'{self.service} {self.region}: {self.address}'


def terraform_str(item) -> str:
res = ""
if (isinstance(item, dict)):
res += '{\n'
for key, value in item.items():
res += f'{key} = {terraform_str(value)}\n'
res += '}'
elif (isinstance(item, list)):
res += '['
for i, value in enumerate(item):
if i != 0:
res += ', '
res += terraform_str(value)
res += ']'
else:
res += f'"{item}"'
return res


projects = {"domain-registry", "domain-registry-sandbox",
"domain-registry-crash", "domain-registry-crash",
"domain-registry-alpha"}

if __name__ == '__main__':
if len(sys.argv) != 2:
raise ValueError('Usage: get-endpoints.py <project>')
project = sys.argv[1]
if (project not in projects):
raise ValueError(f'Invalid project: {project}')
clusters = get_clusters(project)
ips = []
res = {}
for cluster, region in clusters.items():
with UseCluster(cluster, region, project):
for service in ['whois', 'whois-canary', 'epp', 'epp-canary']:
map_key = service.replace('-', '_')
for ip in get_endpoints('services', service,
['status', 'loadBalancer', 'ingress',
'ip']):
ip = ipaddress.ip_address(ip)
if isinstance(ip, IPv4Address):
map_key_with_iptype = map_key + '_ipv4'
else:
map_key_with_iptype = map_key + '_ipv6'
if map_key_with_iptype not in res:
res[map_key_with_iptype] = {}
res[map_key_with_iptype][get_region_symbol(region)] = [ip]
ips.append(IP(service, get_region_symbol(region), ip))
if not region.startswith('us'):
continue
ip = get_endpoints('gateways.gateway.networking.k8s.io', 'nomulus',
['status', 'addresses', 'value'])[0]
print(f'nomulus: {ip}')
res['https_ip'] = ipaddress.ip_address(ip)
ips.sort(key=attrgetter('region'))
ips.sort(key=methodcaller('is_ipv6'))
ips.sort(key=attrgetter('service'))
for ip in ips:
print(ip)
print("Terraform friendly output:")
print(terraform_str(res))
27 changes: 22 additions & 5 deletions jetty/kubernetes/nomulus-frontend.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ spec:
fieldPath: metadata.namespace
- name: CONTAINER_NAME
value: frontend
- name: PROXY_NAME
- name: EPP
image: gcr.io/GCP_PROJECT/proxy
ports:
- containerPort: 30002
Expand All @@ -52,7 +52,7 @@ spec:
fieldRef:
fieldPath: metadata.namespace
- name: CONTAINER_NAME
value: PROXY_NAME
value: EPP
---
# Only need to define the service account once per cluster.
apiVersion: v1
Expand Down Expand Up @@ -92,9 +92,26 @@ spec:
- port: 80
targetPort: http
name: http
- port: 700
targetPort: epp
name: epp
---
apiVersion: v1
kind: Service
metadata:
name: EPP
annotations:
cloud.google.com/l4-rbs: enabled
networking.gke.io/weighted-load-balancing: pods-per-node
spec:
type: LoadBalancer
# Traffic is directly delivered to a node, preserving the original source IP.
externalTrafficPolicy: Local
ipFamilies: [IPv4, IPv6]
ipFamilyPolicy: RequireDualStack
selector:
service: frontend
ports:
- port: 700
targetPort: epp
name: epp
---
apiVersion: net.gke.io/v1
kind: ServiceExport
Expand Down
27 changes: 22 additions & 5 deletions jetty/kubernetes/nomulus-pubapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ spec:
fieldPath: metadata.namespace
- name: CONTAINER_NAME
value: pubapi
- name: PROXY_NAME
- name: WHOIS
image: gcr.io/GCP_PROJECT/proxy
ports:
- containerPort: 30001
Expand All @@ -52,7 +52,7 @@ spec:
fieldRef:
fieldPath: metadata.namespace
- name: CONTAINER_NAME
value: PROXY_NAME
value: WHOIS
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
Expand Down Expand Up @@ -84,9 +84,26 @@ spec:
- port: 80
targetPort: http
name: http
- port: 43
targetPort: whois
name: whois
---
apiVersion: v1
kind: Service
metadata:
name: WHOIS
annotations:
cloud.google.com/l4-rbs: enabled
networking.gke.io/weighted-load-balancing: pods-per-node
spec:
type: LoadBalancer
# Traffic is directly delivered to a node, preserving the original source IP.
externalTrafficPolicy: Local
ipFamilies: [IPv4, IPv6]
ipFamilyPolicy: RequireDualStack
selector:
service: pubapi
ports:
- port: 43
targetPort: whois
name: whois
---
apiVersion: net.gke.io/v1
kind: ServiceExport
Expand Down

0 comments on commit 30137b5

Please sign in to comment.