Skip to content

Commit

Permalink
feat!: Init operator (#154)
Browse files Browse the repository at this point in the history
* operator module

* pass license

* Update main.tf

* fix: Allow for inbound-cidrs for the ALB

* Disabling otel per DD potential Conflict

* Revert "fix: Allow for inbound-cidrs for the ALB"

This reverts commit 3cb558b.

* fixing a bug, can't quote the list of CIDRs

* fxing MR mistake

* Update main.tf

* Add extra envs

* testing tf change

* removing test tf change

* adding efs-csi-driver for weave

* adding efs-csi-driver policy for weave

* adding efs-csi-driver policy for weave and fmting

* namespacing fix

* fixing arn

* updating policy

* updates for weave EFS storage class

* remove debug block

* fix a provider error

* removing name

* updating sg name

* refactor to app-eks

* adding aws_security_group_rule

* adding aws_security_group_rule

* adding aws_security_group_rule

* weird spacing issue

* weird spacing issue

* Fix bucket kms key arn for external buckets

* fixing the username->user typo

---------

Co-authored-by: Zachary Blasczyk <[email protected]>
Co-authored-by: Zachary Blasczyk <[email protected]>
  • Loading branch information
3 people authored Jan 8, 2024
1 parent 9bbabcd commit 95def33
Show file tree
Hide file tree
Showing 14 changed files with 155 additions and 84 deletions.
130 changes: 71 additions & 59 deletions main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -183,62 +183,74 @@ module "redis" {
kms_key_arn = local.kms_key_arn
}

# Comming soon!
# module "wandb" {
# source = "wandb/wandb/helm"
# version = "1.2.0"

# depends_on = [
# module.database,
# module.app_eks,
# module.redis,
# ]

# operator_chart_version = "1.1.0"
# controller_image_tag = "1.10.1"

# spec = {
# values = {
# global = {
# host = local.url
# license = var.license

# bucket = {
# provider = "s3"
# name = local.bucket_name
# region = data.aws_s3_bucket.file_storage.region
# kmsKey = local.kms_key_arn
# }

# mysql = {
# host = module.database.endpoint
# password = module.database.password
# username = module.database.username
# database = module.database.database_name
# port = module.database.port
# }

# redis = {
# host = module.redis.0.host
# port = "${module.redis.0.port}?tls=true"
# }
# }

# ingress = {
# class = "alb"

# annotations = {
# "alb.ingress.kubernetes.io/load-balancer-name" = "${var.namespace}-alb-k8s"
# "alb.ingress.kubernetes.io/inbound-cidrs" = "0.0.0.0/0"
# "alb.ingress.kubernetes.io/scheme" = "internet-facing"
# "alb.ingress.kubernetes.io/target-type" = "ip"
# "alb.ingress.kubernetes.io/listen-ports" = "[{\\\"HTTPS\\\": 443}]"
# "alb.ingress.kubernetes.io/certificate-arn" = local.acm_certificate_arn
# }
# }

# mysql = { install = false }
# redis = { install = false }
# }
# }
# }
module "wandb" {
source = "wandb/wandb/helm"
version = "1.2.0"

depends_on = [
module.database,
module.app_eks,
module.redis,
]
operator_chart_version = "1.1.0"
controller_image_tag = "1.10.1"

spec = {
values = {
global = {
host = local.url
license = var.license

extraEnv = var.other_wandb_env

bucket = {
provider = "s3"
name = local.bucket_name
region = data.aws_s3_bucket.file_storage.region
kmsKey = local.use_external_bucket ? var.bucket_kms_key_arn : local.kms_key_arn
}

mysql = {
host = module.database.endpoint
password = module.database.password
user = module.database.username
database = module.database.database_name
port = module.database.port
}

redis = {
host = module.redis.0.host
port = "${module.redis.0.port}?tls=true"
}
}

ingress = {
class = "alb"

annotations = {
"alb.ingress.kubernetes.io/load-balancer-name" = "${var.namespace}-alb-k8s"
"alb.ingress.kubernetes.io/inbound-cidrs" = <<-EOF
${join("\\,", var.allowed_inbound_cidr)}
EOF
"alb.ingress.kubernetes.io/scheme" = "internet-facing"
"alb.ingress.kubernetes.io/target-type" = "ip"
"alb.ingress.kubernetes.io/listen-ports" = "[{\\\"HTTPS\\\": 443}]"
"alb.ingress.kubernetes.io/certificate-arn" = local.acm_certificate_arn
}
}

mysql = { install = false }
redis = { install = false }

weave = {
persistence = {
provider = "efs"
efs = {
fileSystemId = module.app_eks.efs_id
}

}
}
}
}
}
41 changes: 41 additions & 0 deletions modules/app_eks/efs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
resource "random_pet" "efs" {
length = 2
}

resource "aws_efs_file_system" "storage_class" {
creation_token = "${var.namespace}-${random_pet.efs.id}"
encrypted = true
performance_mode = "generalPurpose"
throughput_mode = "elastic"
}

resource "aws_efs_backup_policy" "storage_class" {
file_system_id = aws_efs_file_system.storage_class.id

backup_policy {
status = "DISABLED"
}
}

resource "aws_security_group" "storage_class_nfs" {
name = "${var.namespace}-${random_pet.efs.id}"
description = "Security group for NFS traffic"
vpc_id = var.network_id
}

resource "aws_security_group_rule" "nfs_ingress" {
description = "NFS inbound"
type = "ingress"
from_port = 2049
to_port = 2049
protocol = "tcp"
security_group_id = aws_security_group.storage_class_nfs.id
source_security_group_id = aws_security_group.primary_workers.id
}

resource "aws_efs_mount_target" "storage_class" {
for_each = { for subnet in var.network_private_subnets : subnet => subnet }
file_system_id = aws_efs_file_system.storage_class.id
subnet_id = each.value
security_groups = [aws_security_group.storage_class_nfs.id]
}
4 changes: 2 additions & 2 deletions modules/app_eks/external_dns/external_dns.tf
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ resource "helm_release" "external_dns" {
}

set {
name = "domainFilters[0]"
name = "domainFilters[0]"
value = var.fqdn
}

set {
name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
value = aws_iam_role.default.arn
}
}
}
2 changes: 1 addition & 1 deletion modules/app_eks/external_dns/variables.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
variable "namespace" {
type = string
type = string
}

variable "oidc_provider" {
Expand Down
2 changes: 1 addition & 1 deletion modules/app_eks/iam-policy-docs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ data "aws_iam_policy_document" "secrets_manager" {
"secretsmanager:GetSecretValue",
"secretsmanager:DeleteSecretVersion"
]
effect = "Allow"
effect = "Allow"
resources = ["arn:aws:secretsmanager:*:${data.aws_caller_identity.current.account_id}:secret:${var.namespace}*"]
}
}
5 changes: 5 additions & 0 deletions modules/app_eks/iam-role-attachments.tf
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ resource "aws_iam_role_policy_attachment" "eks_cni" {
policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
}

resource "aws_iam_role_policy_attachment" "eks_efs" {
role = aws_iam_role.node.name
policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy"
}

resource "aws_iam_role_policy_attachment" "eks_worker_node" {
role = aws_iam_role.node.name
policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
Expand Down
2 changes: 1 addition & 1 deletion modules/app_eks/lb_controller/controller.tf
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ resource "helm_release" "aws_load_balancer_controller" {
repository = "https://aws.github.io/eks-charts"
chart = "aws-load-balancer-controller"
namespace = "kube-system"
version = "1.6.1"
version = "1.6.2"

set {
name = "clusterName"
Expand Down
2 changes: 1 addition & 1 deletion modules/app_eks/lb_controller/variables.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
variable "namespace" {
type = string
type = string
}

variable "oidc_provider" {
Expand Down
18 changes: 10 additions & 8 deletions modules/app_eks/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,16 @@ resource "aws_eks_addon" "eks" {
]
}

resource "aws_eks_addon" "efs" {
cluster_name = module.eks.cluster_id
addon_name = "aws-efs-csi-driver"
addon_version = "v1.7.1-eksbuild.1" # Ensure this version is compatible
resolve_conflicts = "OVERWRITE"
depends_on = [
module.eks
]
}

# removed due to conflict with
# AWS Load Balancer Controller
# being installed with Helm.
Expand All @@ -25,14 +35,6 @@ resource "aws_eks_addon" "eks" {
# depends_on = [module.eks]
#}

locals {
managed_policy_arns = concat([
"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
], var.eks_policy_arns)
}

module "eks" {
source = "terraform-aws-modules/eks/aws"
version = "~> 17.23"
Expand Down
13 changes: 10 additions & 3 deletions modules/app_eks/outputs.tf
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
output "autoscaling_group_names" {
value = { for name, value in module.eks.node_groups : name => lookup(lookup(lookup(value, "resources")[0], "autoscaling_groups")[0], "name") }
}
output "cluster_id" {
value = module.eks.cluster_id
description = "ID of the created EKS cluster"
}

output "autoscaling_group_names" {
value = { for name, value in module.eks.node_groups : name => lookup(lookup(lookup(value, "resources")[0], "autoscaling_groups")[0], "name") }
output "efs_id" {
value = aws_efs_file_system.storage_class.id
}

output "node_role" {
value = aws_iam_role.node
}
}

output "primary_workers_security_group_id" {
value = aws_security_group.primary_workers.id
}
2 changes: 1 addition & 1 deletion modules/file_storage/outputs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,4 @@ output "bucket_queue_name" {

output "bucket_queue_arn" {
value = var.create_queue ? aws_sqs_queue.file_storage.0.arn : null
}
}
4 changes: 2 additions & 2 deletions modules/networking/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ module "vpc" {
single_nat_gateway = false

private_subnet_tags = {
"kubernetes.io/role/internal-elb" = "1"
"kubernetes.io/role/internal-elb" = "1"
}

public_subnet_tags = {
"kubernetes.io/role/elb" = "1"
"kubernetes.io/role/elb" = "1"
}
}
1 change: 0 additions & 1 deletion outputs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,3 @@ output "url" {
value = local.url
description = "The URL to the W&B application"
}

13 changes: 9 additions & 4 deletions variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -327,8 +327,13 @@ variable "elasticache_node_type" {
# ##########################################
# # Weights & Biases #
# ##########################################
# variable "license" {
# type = string
# description = "Weights & Biases license key."
# }
variable "license" {
type = string
description = "Weights & Biases license key."
}

variable "other_wandb_env" {
type = map(any)
description = "Extra environment variables for W&B"
default = {}
}

0 comments on commit 95def33

Please sign in to comment.