Skip to content

Commit

Permalink
Expose EPP and WHOIS endpoints on reginal load balancers
Browse files Browse the repository at this point in the history
k8s does not have a way to expose a global load balancer with TCP
endpoints, and setting up node port-based routing is a chore, even with
Terraform (which is what we did with the standalone proxy).

We will use Cloud DNS's geolocation routing policy to ensure that
clients connect to the endpoint closest to them.
  • Loading branch information
jianglai committed Dec 19, 2024
1 parent 98443fa commit 08477dc
Show file tree
Hide file tree
Showing 4 changed files with 168 additions and 22 deletions.
18 changes: 4 additions & 14 deletions jetty/deploy-nomulus-for-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,27 +37,17 @@ do
sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
sed s/ENVIRONMENT/"${environment}"/g | \
sed s/PROXY_ENV/"${environment}"/g | \
sed s/PROXY_NAME/"proxy"/g | \
sed s/WHOIS_SERVICE/"whois"/g | \
sed s/EPP/"epp"/g | \
sed s/WHOIS/"whois"/g | \
kubectl apply -f -
if [[ "${service}" == pubapi ]]
then
echo "${parts[1]}"
kubectl get services/whois
fi
# canary
sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
sed s/ENVIRONMENT/"${environment}"/g | \
sed s/PROXY_ENV/"${environment}_canary"/g | \
sed s/PROXY_NAME/"proxy-canary"/g | \
sed s/WHOIS_SERVICE/"whois-canary"/g | \
sed s/EPP/"epp-canary"/g | \
sed s/WHOIS/"whois-canary"/g | \
sed s/"${service}"/"${service}-canary"/g | \
kubectl apply -f -
if [[ "${service}" == pubapi ]]
then
echo "${parts[1]}"
kubectl get services/whois-canary
fi
done
# Kills all running pods, new pods created will be pulling the new image.
kubectl delete pods --all
Expand Down
139 changes: 139 additions & 0 deletions jetty/get-endpoints.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
#! /bin/env python3
import ipaddress
import subprocess
from dataclasses import dataclass
from ipaddress import IPv4Address
from ipaddress import IPv6Address
from operator import attrgetter
from operator import methodcaller


class PreserveContext:
def __enter__(self):
self._context = run_command('kubectl config current-context')

def __exit__(self, type, value, traceback):
run_command('kubectl config use-context ' + self._context)


class UseCluster(PreserveContext):
def __init__(self, cluster: str, region: str, project: str):
self._cluster = cluster
self._region = region
self._project = project

def __enter__(self):
super().__enter__()
cmd = f'gcloud container clusters get-credentials {self._cluster} --location {self._region} --project {self._project}'
run_command(cmd)


def run_command(cmd: str, print_output=False) -> str:
proc = subprocess.run(cmd, text=True, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if print_output:
print(proc.stdout)
return proc.stdout


def get_clusters(project: str) -> dict[str, str]:
cmd = 'gcloud container clusters list --project ' + project
lines = run_command(cmd)
res = {}
for line in lines.split('\n'):
if not line.startswith('nomulus-cluster'):
continue
parts = line.split()
res[parts[0]] = parts[1]
return res


def get_endpoints(service: str, resource: str = 'services',
ip_idx: int = 3) -> list[str]:
res = []
lines = run_command(f'kubectl get {resource}/{service}')
for line in lines.split('\n'):
if not line.startswith(service):
continue
res.extend(line.split()[ip_idx].split(','))
return res


def is_ipv6(addr: str) -> bool:
return ':' in addr


def get_region_symbol(region: str) -> str:
if region.startswith('us'):
return 'amer'
if region.startswith('europe'):
return 'emea'
if region.startswith('asia'):
return 'apac'
return 'other'


@dataclass
class IP:
service: str
region: str
address: IPv4Address | IPv6Address

def is_ipv6(self) -> bool:
return self.address.version == 6

def __str__(self) -> str:
return f'{self.service} {self.region}: {self.address}'


def terraform_str(item) -> str:
res = ""
if (isinstance(item, dict)):
res += '{\n'
for key, value in item.items():
res += f'{key} = {terraform_str(value)}\n'
res += '}'
elif (isinstance(item, list)):
res += '['
for i, value in enumerate(item):
if i != 0:
res += ', '
res += terraform_str(value)
res += ']'
else:
res += f'"{item}"'
return res


if __name__ == '__main__':
project = 'domain-registry-alpha'
clusters = get_clusters(project)
ips = []
res = {}
for cluster, region in clusters.items():
with UseCluster(cluster, region, project):
for service in ['whois', 'whois-canary', 'epp', 'epp-canary']:
map_key = service.replace('-', '_')
for ip in get_endpoints(service):
ip = ipaddress.ip_address(ip)
if isinstance(ip, IPv4Address):
map_key_with_iptype = map_key + '_ipv4'
else:
map_key_with_iptype = map_key + '_ipv6'
if map_key_with_iptype not in res:
res[map_key_with_iptype] = {}
res[map_key_with_iptype][get_region_symbol(region)] = [ip]
ips.append(IP(service, get_region_symbol(region), ip))
if not region.startswith('us'):
continue
ip = \
get_endpoints('nomulus', 'gateways.gateway.networking.k8s.io',
2)[0]
print(f'nomulus: {ip}')
res['https_ip'] = ipaddress.ip_address(ip)
ips.sort(key=attrgetter('region'))
ips.sort(key=methodcaller('is_ipv6'))
ips.sort(key=attrgetter('service'))
for ip in ips:
print(ip)
print(terraform_str(res))
27 changes: 22 additions & 5 deletions jetty/kubernetes/nomulus-frontend.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ spec:
fieldPath: metadata.namespace
- name: CONTAINER_NAME
value: frontend
- name: PROXY_NAME
- name: EPP
image: gcr.io/GCP_PROJECT/proxy
ports:
- containerPort: 30002
Expand All @@ -52,7 +52,7 @@ spec:
fieldRef:
fieldPath: metadata.namespace
- name: CONTAINER_NAME
value: PROXY_NAME
value: EPP
---
# Only need to define the service account once per cluster.
apiVersion: v1
Expand Down Expand Up @@ -92,9 +92,26 @@ spec:
- port: 80
targetPort: http
name: http
- port: 700
targetPort: epp
name: epp
---
apiVersion: v1
kind: Service
metadata:
name: EPP
annotations:
cloud.google.com/l4-rbs: enabled
networking.gke.io/weighted-load-balancing: pods-per-node
spec:
type: LoadBalancer
# Traffic is directly delivered to a node, preserving the original source IP.
externalTrafficPolicy: Local
ipFamilies: [IPv4, IPv6]
ipFamilyPolicy: RequireDualStack
selector:
service: frontend
ports:
- port: 700
targetPort: epp
name: epp
---
apiVersion: net.gke.io/v1
kind: ServiceExport
Expand Down
6 changes: 3 additions & 3 deletions jetty/kubernetes/nomulus-pubapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ spec:
fieldPath: metadata.namespace
- name: CONTAINER_NAME
value: pubapi
- name: PROXY_NAME
- name: WHOIS
image: gcr.io/GCP_PROJECT/proxy
ports:
- containerPort: 30001
Expand All @@ -52,7 +52,7 @@ spec:
fieldRef:
fieldPath: metadata.namespace
- name: CONTAINER_NAME
value: PROXY_NAME
value: WHOIS
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
Expand Down Expand Up @@ -88,7 +88,7 @@ spec:
apiVersion: v1
kind: Service
metadata:
name: WHOIS_SERVICE
name: WHOIS
annotations:
cloud.google.com/l4-rbs: enabled
networking.gke.io/weighted-load-balancing: pods-per-node
Expand Down

0 comments on commit 08477dc

Please sign in to comment.