From b37dfb8514a1853bc7a07353a0fba744310a8400 Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Sun, 24 Sep 2023 21:09:53 +1000 Subject: [PATCH 01/20] feat: decouple secret management from terraform (#300) --- .github/workflows/terraform-dev.yml | 15 ++-- .github/workflows/terraform-feature-env.yml | 10 ++- infra/main.tf | 91 +++++++++++++++++---- infra/modules/leader/main.tf | 60 ++++---------- infra/modules/leader/variables.tf | 28 +++---- infra/modules/signer/main.tf | 89 ++++++-------------- infra/modules/signer/variables.tf | 16 ++-- infra/partner/main.tf | 51 +++++++++--- infra/partner/template.tfvars | 11 +++ infra/partner/variables.tf | 8 +- infra/terraform-dev.tfvars | 45 ++++------ infra/variables.tf | 44 ++++------ 12 files changed, 247 insertions(+), 221 deletions(-) create mode 100644 infra/partner/template.tfvars diff --git a/.github/workflows/terraform-dev.yml b/.github/workflows/terraform-dev.yml index 60a58ed41..7ac9e63fc 100644 --- a/.github/workflows/terraform-dev.yml +++ b/.github/workflows/terraform-dev.yml @@ -53,17 +53,19 @@ jobs: env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} + # Build Docker image. + - name: Docker Image + id: build + run: docker build .. -t near/mpc-recovery + # Generates an execution plan for Terraform - name: Terraform Plan id: plan run: | terraform plan -input=false -no-color -lock-timeout=1h -var-file terraform-dev.tfvars \ - -var "credentials=$GOOGLE_CREDENTIALS" \ - -var "account_creator_id=mpc-recovery-dev-creator.testnet" \ - -var "account_creator_sk=$ACCOUNT_CREATOR_SK" + -var "credentials=$GOOGLE_CREDENTIALS" env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} - ACCOUNT_CREATOR_SK: ${{ secrets.ACCOUNT_CREATOR_SK_DEV }} - uses: actions/github-script@v6 if: github.event_name == 'pull_request' @@ -137,9 +139,6 @@ jobs: if: github.ref == 'refs/heads/develop' && github.event_name == 'push' run: | terraform apply -auto-approve -input=false -lock-timeout=1h -var-file terraform-dev.tfvars \ - -var "credentials=$GOOGLE_CREDENTIALS" \ - -var "account_creator_id=mpc-recovery-dev-creator.testnet" \ - -var "account_creator_sk=$ACCOUNT_CREATOR_SK" + -var "credentials=$GOOGLE_CREDENTIALS" env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} - ACCOUNT_CREATOR_SK: ${{ secrets.ACCOUNT_CREATOR_SK_DEV }} diff --git a/.github/workflows/terraform-feature-env.yml b/.github/workflows/terraform-feature-env.yml index 03a5c5754..a4618fa8b 100644 --- a/.github/workflows/terraform-feature-env.yml +++ b/.github/workflows/terraform-feature-env.yml @@ -41,10 +41,18 @@ jobs: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} PR_NUMBER: ${{ env.PR_NUMBER }} + # Build Docker image. + - name: Docker Image + id: build + run: docker build .. -t near/mpc-recovery + # Applies Terraform configuration to the temporary environment - name: Terraform Apply id: apply - run: terraform apply -auto-approve -input=false -no-color -lock-timeout=1h -var-file terraform-dev.tfvars -var "credentials=$GOOGLE_CREDENTIALS" -var "env=dev-$PR_NUMBER" + run: | + terraform apply -auto-approve -input=false -no-color -lock-timeout=1h -var-file terraform-dev.tfvars \ + -var "credentials=$GOOGLE_CREDENTIALS" \ + -var "env=dev-$PR_NUMBER" env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} PR_NUMBER: ${{ env.PR_NUMBER }} diff --git a/infra/main.tf b/infra/main.tf index 54e4fa9cd..5d4774858 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -56,6 +56,9 @@ provider "docker" { } } +/* + * Create brand new service account with basic IAM + */ resource "google_service_account" "service_account" { account_id = "mpc-recovery-${var.env}" display_name = "MPC Recovery ${var.env} Account" @@ -76,25 +79,66 @@ resource "google_project_iam_member" "service-account-datastore-user" { member = "serviceAccount:${google_service_account.service_account.email}" } +/* + * Ensure service account has access to Secret Manager variables + */ +resource "google_secret_manager_secret_iam_member" "cipher_key_secret_access" { + count = length(var.signer_configs) + + secret_id = var.signer_configs[count.index].cipher_key_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "secret_share_secret_access" { + count = length(var.signer_configs) + + secret_id = var.signer_configs[count.index].sk_share_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "oidc_providers_secret_access" { + secret_id = var.oidc_providers_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "account_creator_secret_access" { + secret_id = var.account_creator_sk_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "fast_auth_partners_secret_access" { + secret_id = var.fast_auth_partners_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +/* + * Create Artifact Registry repo, tag existing Docker image and push to the repo + */ resource "google_artifact_registry_repository" "mpc_recovery" { repository_id = "mpc-recovery-${var.env}" format = "DOCKER" } resource "docker_registry_image" "mpc_recovery" { - name = docker_image.mpc_recovery.name + name = docker_tag.mpc_recovery.target_image keep_remotely = true } -resource "docker_image" "mpc_recovery" { - name = "${var.region}-docker.pkg.dev/${var.project}/${google_artifact_registry_repository.mpc_recovery.name}/mpc-recovery-${var.env}:${data.external.git_checkout.result.sha}" - build { - context = "${path.cwd}/.." - } +resource "docker_tag" "mpc_recovery" { + source_image = var.docker_image + target_image = "${var.region}-docker.pkg.dev/${var.project}/${google_artifact_registry_repository.mpc_recovery.name}/mpc-recovery-${var.env}:${data.external.git_checkout.result.sha}" } +/* + * Create multiple signer nodes + */ module "signer" { - count = length(var.cipher_keys) + count = length(var.signer_configs) source = "./modules/signer" env = var.env @@ -102,17 +146,25 @@ module "signer" { region = var.region zone = var.zone service_account_email = google_service_account.service_account.email - docker_image = docker_image.mpc_recovery.name + docker_image = docker_tag.mpc_recovery.target_image - node_id = count.index - oidc_providers = var.oidc_providers + node_id = count.index - cipher_key = var.cipher_keys[count.index] - sk_share = var.sk_shares[count.index] + oidc_providers_secret_id = var.oidc_providers_secret_id + cipher_key_secret_id = var.signer_configs[count.index].cipher_key_secret_id + sk_share_secret_id = var.signer_configs[count.index].sk_share_secret_id - depends_on = [docker_registry_image.mpc_recovery] + depends_on = [ + docker_registry_image.mpc_recovery, + google_secret_manager_secret_iam_member.cipher_key_secret_access, + google_secret_manager_secret_iam_member.secret_share_secret_access, + google_secret_manager_secret_iam_member.oidc_providers_secret_access + ] } +/* + * Create leader node + */ module "leader" { source = "./modules/leader" @@ -121,15 +173,20 @@ module "leader" { region = var.region zone = var.zone service_account_email = google_service_account.service_account.email - docker_image = docker_image.mpc_recovery.name + docker_image = docker_tag.mpc_recovery.target_image signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) near_rpc = local.workspace.near_rpc near_root_account = local.workspace.near_root_account account_creator_id = var.account_creator_id - fast_auth_partners = var.fast_auth_partners - account_creator_sk = var.account_creator_sk + account_creator_sk_secret_id = var.account_creator_sk_secret_id + fast_auth_partners_secret_id = var.fast_auth_partners_secret_id - depends_on = [docker_registry_image.mpc_recovery, module.signer] + depends_on = [ + docker_registry_image.mpc_recovery, + google_secret_manager_secret_iam_member.account_creator_secret_access, + google_secret_manager_secret_iam_member.fast_auth_partners_secret_access, + module.signer + ] } diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf index 5b8895949..e26a67790 100644 --- a/infra/modules/leader/main.tf +++ b/infra/modules/leader/main.tf @@ -1,39 +1,3 @@ -resource "google_secret_manager_secret" "account_creator_sk" { - secret_id = "mpc-recovery-account-creator-sk-${var.env}" - replication { - automatic = true - } -} - -resource "google_secret_manager_secret_version" "account_creator_sk_data" { - secret = google_secret_manager_secret.account_creator_sk.name - secret_data = var.account_creator_sk -} - -resource "google_secret_manager_secret_iam_member" "account_creator_secret_access" { - secret_id = google_secret_manager_secret.account_creator_sk.id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${var.service_account_email}" -} - -resource "google_secret_manager_secret" "fast_auth_partners" { - secret_id = "mpc-recovery-allowed-oidc-providers-leader-${var.env}" - replication { - automatic = true - } -} - -resource "google_secret_manager_secret_version" "fast_auth_partners_data" { - secret = google_secret_manager_secret.fast_auth_partners.name - secret_data = jsonencode(var.fast_auth_partners) -} - -resource "google_secret_manager_secret_iam_member" "fast_auth_partners_secret_access" { - secret_id = google_secret_manager_secret.fast_auth_partners.id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${var.service_account_email}" -} - resource "google_cloud_run_v2_service" "leader" { name = "mpc-recovery-leader-${var.env}" location = var.region @@ -79,6 +43,24 @@ resource "google_cloud_run_v2_service" "leader" { name = "MPC_RECOVERY_ENV" value = var.env } + env { + name = "MPC_RECOVERY_ACCOUNT_CREATOR_SK" + value_source { + secret_key_ref { + secret = var.account_creator_sk_secret_id + version = "latest" + } + } + } + env { + name = "FAST_AUTH_PARTNERS" + value_source { + secret_key_ref { + secret = var.fast_auth_partners_secret_id + version = "latest" + } + } + } env { name = "RUST_LOG" value = "mpc_recovery=debug" @@ -98,12 +80,6 @@ resource "google_cloud_run_v2_service" "leader" { } } } - depends_on = [ - google_secret_manager_secret_version.account_creator_sk_data, - google_secret_manager_secret_version.fast_auth_partners_data, - google_secret_manager_secret_iam_member.account_creator_secret_access, - google_secret_manager_secret_iam_member.fast_auth_partners_secret_access - ] } // Allow unauthenticated requests diff --git a/infra/modules/leader/variables.tf b/infra/modules/leader/variables.tf index b9cd6e9a4..816ab1cd2 100644 --- a/infra/modules/leader/variables.tf +++ b/infra/modules/leader/variables.tf @@ -1,19 +1,25 @@ variable "env" { + type = string } variable "project" { + type = string } variable "region" { + type = string } variable "zone" { + type = string } variable "service_account_email" { + type = string } variable "docker_image" { + type = string } # Application variables @@ -22,28 +28,22 @@ variable "signer_node_urls" { } variable "near_rpc" { + type = string } variable "near_root_account" { + type = string } variable "account_creator_id" { + type = string } -variable "fast_auth_partners" { - type = list(object({ - oidc_provider = object({ - issuer = string - audience = string - }) - relayer = object({ - url = string - api_key = string - }) - })) - default = [] +# Secrets +variable "account_creator_sk_secret_id" { + type = string } -# Secrets -variable "account_creator_sk" { +variable "fast_auth_partners_secret_id" { + type = string } diff --git a/infra/modules/signer/main.tf b/infra/modules/signer/main.tf index 2d165a065..120758736 100644 --- a/infra/modules/signer/main.tf +++ b/infra/modules/signer/main.tf @@ -1,57 +1,3 @@ -resource "google_secret_manager_secret" "cipher_key" { - secret_id = "mpc-recovery-encryption-cipher-${var.node_id}-${var.env}" - replication { - automatic = true - } -} - -resource "google_secret_manager_secret_version" "cipher_key_data" { - secret = google_secret_manager_secret.cipher_key.name - secret_data = var.cipher_key -} - -resource "google_secret_manager_secret_iam_member" "cipher_key_secret_access" { - secret_id = google_secret_manager_secret.cipher_key.id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${var.service_account_email}" -} - -resource "google_secret_manager_secret" "secret_share" { - secret_id = "mpc-recovery-secret-share-${var.node_id}-${var.env}" - replication { - automatic = true - } -} - -resource "google_secret_manager_secret_version" "secret_share_data" { - secret = google_secret_manager_secret.secret_share.name - secret_data = var.sk_share -} - -resource "google_secret_manager_secret_iam_member" "secret_share_secret_access" { - secret_id = google_secret_manager_secret.secret_share.id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${var.service_account_email}" -} - -resource "google_secret_manager_secret" "oidc_providers" { - secret_id = "mpc-recovery-allowed-oidc-providers-${var.node_id}-${var.env}" - replication { - automatic = true - } -} - -resource "google_secret_manager_secret_version" "oidc_providers_data" { - secret = google_secret_manager_secret.oidc_providers.name - secret_data = jsonencode(var.oidc_providers) -} - -resource "google_secret_manager_secret_iam_member" "oidc_providers_secret_access" { - secret_id = google_secret_manager_secret.oidc_providers.id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${var.service_account_email}" -} - resource "google_cloud_run_v2_service" "signer" { name = "mpc-recovery-signer-${var.node_id}-${var.env}" location = var.region @@ -85,6 +31,33 @@ resource "google_cloud_run_v2_service" "signer" { name = "MPC_RECOVERY_ENV" value = var.env } + env { + name = "MPC_RECOVERY_CIPHER_KEY" + value_source { + secret_key_ref { + secret = var.cipher_key_secret_id + version = "latest" + } + } + } + env { + name = "MPC_RECOVERY_SK_SHARE" + value_source { + secret_key_ref { + secret = var.sk_share_secret_id + version = "latest" + } + } + } + env { + name = "OIDC_PROVIDERS" + value_source { + secret_key_ref { + secret = var.oidc_providers_secret_id + version = "latest" + } + } + } env { name = "RUST_LOG" value = "mpc_recovery=debug" @@ -104,14 +77,6 @@ resource "google_cloud_run_v2_service" "signer" { } } } - depends_on = [ - google_secret_manager_secret_version.cipher_key_data, - google_secret_manager_secret_version.secret_share_data, - google_secret_manager_secret_version.oidc_providers_data, - google_secret_manager_secret_iam_member.cipher_key_secret_access, - google_secret_manager_secret_iam_member.secret_share_secret_access, - google_secret_manager_secret_iam_member.oidc_providers_secret_access - ] } // Allow unauthenticated requests diff --git a/infra/modules/signer/variables.tf b/infra/modules/signer/variables.tf index 4c4567294..2be3c3a64 100644 --- a/infra/modules/signer/variables.tf +++ b/infra/modules/signer/variables.tf @@ -20,17 +20,15 @@ variable "docker_image" { variable "node_id" { } -variable "oidc_providers" { - type = list(object({ - issuer = string - audience = string - })) - default = [] +# Secrets +variable "cipher_key_secret_id" { + type = string } -# Secrets -variable "cipher_key" { +variable "sk_share_secret_id" { + type = string } -variable "sk_share" { +variable "oidc_providers_secret_id" { + type = string } diff --git a/infra/partner/main.tf b/infra/partner/main.tf index 58dce56e0..f35f8a365 100644 --- a/infra/partner/main.tf +++ b/infra/partner/main.tf @@ -33,6 +33,9 @@ provider "docker" { } } +/* + * Create brand new service account with basic IAM + */ resource "google_service_account" "service_account" { account_id = "mpc-recovery-${var.env}" display_name = "MPC Recovery ${var.env} Account" @@ -56,8 +59,32 @@ resource "google_project_iam_binding" "service-account-datastore-user" { ] } +/* + * Ensure service account has access to Secret Manager variables + */ +resource "google_secret_manager_secret_iam_member" "cipher_key_secret_access" { + secret_id = var.cipher_key_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "secret_share_secret_access" { + secret_id = var.sk_share_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "oidc_providers_secret_access" { + secret_id = var.oidc_providers_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +/* + * Create Artifact Registry repo, tag existing Docker image and push to the repo + */ resource "google_artifact_registry_repository" "mpc_recovery" { - repository_id = "mpc-recovery-signer-${var.env}" + repository_id = "mpc-recovery-partner-${var.env}" format = "DOCKER" } @@ -71,13 +98,9 @@ resource "docker_tag" "mpc_recovery" { target_image = "${var.region}-docker.pkg.dev/${var.project}/${google_artifact_registry_repository.mpc_recovery.name}/mpc-recovery-${var.env}" } -# resource "docker_image" "mpc_recovery" { -# name = "${var.region}-docker.pkg.dev/${var.project}/${google_artifact_registry_repository.mpc_recovery.name}/mpc-recovery-${var.env}" -# build { -# context = "${path.cwd}/.." -# } -# } - +/* + * Create a partner signer node + */ module "signer" { source = "../modules/signer" @@ -90,8 +113,14 @@ module "signer" { node_id = var.node_id - cipher_key = var.cipher_key - sk_share = var.sk_share + cipher_key_secret_id = var.cipher_key_secret_id + sk_share_secret_id = var.sk_share_secret_id + oidc_providers_secret_id = var.oidc_providers_secret_id - depends_on = [docker_registry_image.mpc_recovery] + depends_on = [ + docker_registry_image.mpc_recovery, + google_secret_manager_secret_iam_member.cipher_key_secret_access, + google_secret_manager_secret_iam_member.secret_share_secret_access, + google_secret_manager_secret_iam_member.oidc_providers_secret_access + ] } diff --git a/infra/partner/template.tfvars b/infra/partner/template.tfvars new file mode 100644 index 000000000..a9a77bf3b --- /dev/null +++ b/infra/partner/template.tfvars @@ -0,0 +1,11 @@ +env = "partner-dev" +project = "pagoda-discovery-platform-dev" +region = "us-east1" +zone = "us-east1-c" + +docker_image = "near/mpc-recovery" +node_id = "0" + +oidc_providers_secret_id = "mpc-recovery-allowed-oidc-providers-0-dev" +cipher_key_secret_id = "mpc-recovery-encryption-cipher-0-dev" +sk_share_secret_id = "mpc-recovery-secret-share-0-dev" diff --git a/infra/partner/variables.tf b/infra/partner/variables.tf index 848ed498b..83c91c32d 100644 --- a/infra/partner/variables.tf +++ b/infra/partner/variables.tf @@ -21,10 +21,14 @@ variable "node_id" { } # Secrets -variable "cipher_key" { +variable "cipher_key_secret_id" { type = string } -variable "sk_share" { +variable "sk_share_secret_id" { + type = string +} + +variable "oidc_providers_secret_id" { type = string } diff --git a/infra/terraform-dev.tfvars b/infra/terraform-dev.tfvars index 9159b36bd..4f87e3ffb 100644 --- a/infra/terraform-dev.tfvars +++ b/infra/terraform-dev.tfvars @@ -1,33 +1,22 @@ -env = "dev" -project = "pagoda-discovery-platform-dev" +env = "dev" +project = "pagoda-discovery-platform-dev" +docker_image = "near/mpc-recovery" -account_creator_id = "tmp_acount_creator.serhii.testnet" -account_creator_sk = "ed25519:5pFJN3czPAHFWHZYjD4oTtnJE7PshLMeTkSU7CmWkvLaQWchCLgXGF1wwcJmh2AQChGH85EwcL5VW7tUavcAZDSG" -cipher_keys = ["ea28abd17cb76924f62c99f6fd240985c16b9dc85187760c1487e64689d447f5", "cc7a448b28b2a58bada59770b6418ae75ade177abad216385e012805b9cfc8f9", "78be23c9400f4414c043aa966b51e44b3fa3ab790a1779d370d40589a7b02dd2"] -sk_shares = [ - "{\"public_key\":{\"curve\":\"ed25519\",\"point\":[44,250,33,208,230,210,1,232,218,250,54,239,72,81,92,99,10,169,178,160,155,203,106,27,68,188,121,148,143,199,6,241]},\"expanded_private_key\":{\"prefix\":{\"curve\":\"ed25519\",\"scalar\":[102,223,208,90,184,101,17,59,89,36,9,226,244,136,59,225,17,226,66,187,72,197,17,71,28,28,128,125,122,248,32,105]},\"private_key\":{\"curve\":\"ed25519\",\"scalar\":[240,196,61,168,214,169,50,27,103,54,246,131,195,119,194,74,24,183,7,164,92,165,213,35,130,63,118,52,70,141,108,97]}}}", - "{\"public_key\":{\"curve\":\"ed25519\",\"point\":[46,181,130,13,164,112,16,130,63,196,212,83,38,63,120,124,0,35,238,100,212,32,46,7,233,221,2,16,20,189,198,167]},\"expanded_private_key\":{\"prefix\":{\"curve\":\"ed25519\",\"scalar\":[35,145,79,79,99,72,33,94,114,179,89,56,252,168,145,28,195,10,230,89,247,39,194,127,202,75,119,182,59,120,144,83]},\"private_key\":{\"curve\":\"ed25519\",\"scalar\":[88,71,177,97,38,226,233,158,49,168,14,146,117,128,240,16,97,35,56,137,0,69,150,237,4,210,81,35,0,44,233,98]}}}", - "{\"public_key\":{\"curve\":\"ed25519\",\"point\":[226,221,12,58,210,76,171,11,139,88,242,44,18,207,126,120,5,90,208,108,4,93,19,188,24,172,130,61,51,94,10,34]},\"expanded_private_key\":{\"prefix\":{\"curve\":\"ed25519\",\"scalar\":[72,32,251,204,100,91,164,82,140,231,84,166,176,30,167,99,107,71,71,195,83,40,241,205,6,89,122,227,140,146,82,4]},\"private_key\":{\"curve\":\"ed25519\",\"scalar\":[8,248,184,114,40,88,141,189,156,115,215,171,36,210,85,189,12,217,176,9,208,28,141,207,18,18,57,230,231,14,118,116]}}}" -] - -// For leader node -fast_auth_partners = [ +account_creator_id = "mpc-recovery-dev-creator.testnet" +account_creator_sk_secret_id = "mpc-account-creator-sk-dev" +oidc_providers_secret_id = "mpc-allowed-oidc-providers-dev" +fast_auth_partners_secret_id = "mpc-fast-auth-partners-dev" +signer_configs = [ { - oidc_provider = { - issuer = "https://securetoken.google.com/pagoda-oboarding-dev", - audience = "pagoda-oboarding-dev" - }, - relayer = { - url = "http://34.70.226.83:3030", - api_key = null, - }, - } -] - -// For signing nodes -oidc_providers = [ + cipher_key_secret_id = "mpc-cipher-0-dev" + sk_share_secret_id = "mpc-sk-share-0-dev" + }, + { + cipher_key_secret_id = "mpc-cipher-1-dev" + sk_share_secret_id = "mpc-sk-share-1-dev" + }, { - issuer = "https://securetoken.google.com/pagoda-oboarding-dev", - audience = "pagoda-oboarding-dev" + cipher_key_secret_id = "mpc-cipher-2-dev" + sk_share_secret_id = "mpc-sk-share-2-dev" } ] diff --git a/infra/variables.tf b/infra/variables.tf index 5d9e35848..9978857f9 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -20,46 +20,36 @@ variable "zone" { default = "us-east1-c" } +variable "docker_image" { + type = string +} + # Application variables variable "account_creator_id" { default = "tmp_acount_creator.serhii.testnet" } -variable "fast_auth_partners" { - type = list(object({ - oidc_provider = object({ - issuer = string - audience = string - }) - relayer = object({ - url = string - api_key = string - }) - })) - default = [] -} - -variable "oidc_providers" { - type = list(object({ - issuer = string - audience = string - })) - default = [] -} - variable "external_signer_node_urls" { type = list(string) default = [] } # Secrets -variable "account_creator_sk" { +variable "account_creator_sk_secret_id" { + type = string } -variable "cipher_keys" { - type = list(string) +variable "oidc_providers_secret_id" { + type = string } -variable "sk_shares" { - type = list(string) +variable "fast_auth_partners_secret_id" { + type = string +} + +variable "signer_configs" { + type = list(object({ + cipher_key_secret_id = string + sk_share_secret_id = string + })) } From 7997374884c6788a2a907f7ab6944a03a8c4ab36 Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Thu, 12 Oct 2023 10:01:37 -0600 Subject: [PATCH 02/20] initial commit for internal LB --- infra/main.tf | 23 +++++++++ infra/modules/internal_cloudrun_lb/main.tf | 50 +++++++++++++++++++ .../modules/internal_cloudrun_lb/variables.tf | 29 +++++++++++ infra/modules/leader/main.tf | 24 +++++++++ infra/modules/leader/variables.tf | 10 ++++ infra/modules/signer/main.tf | 26 ++++++++++ infra/modules/signer/variables.tf | 10 ++++ 7 files changed, 172 insertions(+) create mode 100644 infra/modules/internal_cloudrun_lb/main.tf create mode 100644 infra/modules/internal_cloudrun_lb/variables.tf diff --git a/infra/main.tf b/infra/main.tf index abd2c6fd6..aae36667e 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -104,6 +104,27 @@ resource "google_secret_manager_secret_iam_member" "fast_auth_partners_secret_ac member = "serviceAccount:${google_service_account.service_account.email}" } +module "mpc-signer-lb" { + + count = length(var.signer_configs) + source = "./modules/internal_cloudrun_lb" + name = "mpc-${var.env}-signer-${count.index}" + network_id = var.env + subnetwork_id = "${var.env}-us-central1" + project_id = var.project + region = "us-central1" + service_name = module.signer.google_cloud_run_v2_service.name +} + +module "mpc-leader-lb" { + source = "./modules/internal_cloudrun_lb" + name = "mpc-${var.env}-leader" + network_id = var.env + subnetwork_id = "${var.env}-us-central1" + project_id = var.project + region = "us-central1" + service_name = module.leader.google_cloud_run_v2_service.name +} /* * Create multiple signer nodes */ @@ -117,6 +138,7 @@ module "signer" { zone = var.zone service_account_email = google_service_account.service_account.email docker_image = var.docker_image + connector_id = "projects/pagoda-shared-infrastructure/locations/us-central1/connectors/${var.env}-connector" node_id = count.index @@ -143,6 +165,7 @@ module "leader" { zone = var.zone service_account_email = google_service_account.service_account.email docker_image = var.docker_image + connector_id = "projects/pagoda-shared-infrastructure/locations/us-central1/connectors/${var.env}-connector" signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) near_rpc = local.workspace.near_rpc diff --git a/infra/modules/internal_cloudrun_lb/main.tf b/infra/modules/internal_cloudrun_lb/main.tf new file mode 100644 index 000000000..f6492bf7a --- /dev/null +++ b/infra/modules/internal_cloudrun_lb/main.tf @@ -0,0 +1,50 @@ +resource "google_compute_region_network_endpoint_group" "default_neg" { + name = "${var.name}-neg" + project = var.project_id + network_endpoint_type = "SERVERLESS" + region = var.region + cloud_run { + service = var.service_name + } +} + +resource "google_compute_region_backend_service" "default" { + name = "${var.name}-backend-service" + project = var.project_id + region = var.region + protocol = "HTTP" + load_balancing_scheme = "INTERNAL_MANAGED" + timeout_sec = 30 + backend { + group = google_compute_region_network_endpoint_group.default_neg.id + balancing_mode = "UTILIZATION" + capacity_scaler = 1.0 + } +} + +resource "google_compute_region_url_map" "default" { + name = "${var.name}-url-map" + project = var.project_id + region = var.region + default_service = google_compute_region_backend_service.default.id +} + +resource "google_compute_region_target_http_proxy" "default" { + name = "${var.name}-http-proxy" + region = var.region + project = var.project_id + url_map = google_compute_region_url_map.default.id +} + +resource "google_compute_forwarding_rule" "default" { + name = "${var.name}-forwarding-rule" + project = var.project_id + region = var.region + ip_protocol = "TCP" + load_balancing_scheme = "INTERNAL_MANAGED" + port_range = "80" + target = google_compute_region_target_http_proxy.default.id + network = var.network_id + subnetwork = var.subnetwork_id + network_tier = "PREMIUM" +} diff --git a/infra/modules/internal_cloudrun_lb/variables.tf b/infra/modules/internal_cloudrun_lb/variables.tf new file mode 100644 index 000000000..c09e1d886 --- /dev/null +++ b/infra/modules/internal_cloudrun_lb/variables.tf @@ -0,0 +1,29 @@ +variable "name" { + type = string + description = "The name to use as prefix for load balancer resources." +} + +variable "service_name" { + type = string + description = "The cloud run service name" +} + +variable "project_id" { + type = string + description = "The GCP project these resources belong to" +} + +variable "region" { + type = string + description = "The region where resources will live." +} + +variable "network_id" { + type = string + description = "The VPC network to connect to." +} + +variable "subnetwork_id" { + type = string + description = "Subnet for hosting the load balancer." +} \ No newline at end of file diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf index e26a67790..2e5b284b6 100644 --- a/infra/modules/leader/main.tf +++ b/infra/modules/leader/main.tf @@ -6,6 +6,13 @@ resource "google_cloud_run_v2_service" "leader" { template { service_account = var.service_account_email + annotations = var.metadata_annotations == null ? null : var.metadata_annotations + + vpc_access { + connector = var.connector_id + egress = "ALL_TRAFFIC" + } + scaling { min_instance_count = 1 max_instance_count = 1 @@ -80,6 +87,23 @@ resource "google_cloud_run_v2_service" "leader" { } } } + + lifecycle { + # List of fields we don't want to see a diff for in terraform. Most of these fields are set + # by GCP and is metadata we don't want to account when considering changes in the service. + ignore_changes = [ + metadata[0].annotations["client.knative.dev/user-image"], + metadata[0].annotations["run.googleapis.com/client-name"], + metadata[0].annotations["run.googleapis.com/client-version"], + metadata[0].annotations["run.googleapis.com/launch-stage"], + metadata[0].annotations["run.googleapis.com/operation-id"], + template[0].metadata[0].annotations["client.knative.dev/user-image"], + template[0].metadata[0].annotations["run.googleapis.com/client-version"], + template[0].metadata[0].annotations["run.googleapis.com/client-name"], + template[0].metadata[0].labels["client.knative.dev/nonce"], + template[0].metadata[0].labels["run.googleapis.com/startupProbeType"], + ] + } } // Allow unauthenticated requests diff --git a/infra/modules/leader/variables.tf b/infra/modules/leader/variables.tf index 816ab1cd2..351750082 100644 --- a/infra/modules/leader/variables.tf +++ b/infra/modules/leader/variables.tf @@ -6,6 +6,16 @@ variable "project" { type = string } +variable "connector_id" { + description = "VPC connector ID for internal traffic" +} + +variable "metadata_annotations" { + type = map(any) + default = null + description = "Annotations for the metadata associated with this Service." +} + variable "region" { type = string } diff --git a/infra/modules/signer/main.tf b/infra/modules/signer/main.tf index 120758736..45c94ff74 100644 --- a/infra/modules/signer/main.tf +++ b/infra/modules/signer/main.tf @@ -6,6 +6,13 @@ resource "google_cloud_run_v2_service" "signer" { template { service_account = var.service_account_email + annotations = var.metadata_annotations == null ? null : var.metadata_annotations + + vpc_access { + connector = var.connector_id + egress = "ALL_TRAFFIC" + } + scaling { min_instance_count = 1 max_instance_count = 1 @@ -67,6 +74,8 @@ resource "google_cloud_run_v2_service" "signer" { container_port = 3000 } + + resources { cpu_idle = false @@ -77,6 +86,23 @@ resource "google_cloud_run_v2_service" "signer" { } } } + + lifecycle { + # List of fields we don't want to see a diff for in terraform. Most of these fields are set + # by GCP and is metadata we don't want to account when considering changes in the service. + ignore_changes = [ + metadata[0].annotations["client.knative.dev/user-image"], + metadata[0].annotations["run.googleapis.com/client-name"], + metadata[0].annotations["run.googleapis.com/client-version"], + metadata[0].annotations["run.googleapis.com/launch-stage"], + metadata[0].annotations["run.googleapis.com/operation-id"], + template[0].metadata[0].annotations["client.knative.dev/user-image"], + template[0].metadata[0].annotations["run.googleapis.com/client-version"], + template[0].metadata[0].annotations["run.googleapis.com/client-name"], + template[0].metadata[0].labels["client.knative.dev/nonce"], + template[0].metadata[0].labels["run.googleapis.com/startupProbeType"], + ] + } } // Allow unauthenticated requests diff --git a/infra/modules/signer/variables.tf b/infra/modules/signer/variables.tf index 2be3c3a64..a1d5aaf24 100644 --- a/infra/modules/signer/variables.tf +++ b/infra/modules/signer/variables.tf @@ -16,6 +16,16 @@ variable "service_account_email" { variable "docker_image" { } +variable "connector_id" { + description = "VPC connector ID for internal traffic" +} + +variable "metadata_annotations" { + type = map(any) + default = null + description = "Annotations for the metadata associated with this Service." +} + # Application variables variable "node_id" { } From 60c8807ed22ad6e16a3090a1d0e7686cb6f3f21c Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Tue, 17 Oct 2023 10:20:51 -0600 Subject: [PATCH 03/20] added working dynamic TF for ILBs --- infra/main.tf | 17 +++++++++-------- infra/modules/leader/main.tf | 17 ----------------- infra/modules/signer/main.tf | 17 ----------------- infra/terraform-dev.tfvars | 2 +- infra/variables.tf | 30 ++++++++++++++++++++++++++++++ 5 files changed, 40 insertions(+), 43 deletions(-) diff --git a/infra/main.tf b/infra/main.tf index aae36667e..f7438c315 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -1,5 +1,6 @@ terraform { backend "gcs" { + bucket = "mpc-recovery-terraform-dev" prefix = "state/mpc-recovery" } @@ -109,21 +110,21 @@ module "mpc-signer-lb" { count = length(var.signer_configs) source = "./modules/internal_cloudrun_lb" name = "mpc-${var.env}-signer-${count.index}" - network_id = var.env - subnetwork_id = "${var.env}-us-central1" + network_id = var.env == "dev" ? data.google_compute_network.dev_network.id : data.google_compute_network.prod_network.id + subnetwork_id = var.env == "dev" ? data.google_compute_subnetwork.dev_subnetwork.id : data.google_compute_subnetwork.prod_subnetwork.id project_id = var.project region = "us-central1" - service_name = module.signer.google_cloud_run_v2_service.name + service_name = "mpc-recovery-signer-${count.index}-${var.env}" } module "mpc-leader-lb" { source = "./modules/internal_cloudrun_lb" name = "mpc-${var.env}-leader" - network_id = var.env - subnetwork_id = "${var.env}-us-central1" + network_id = var.env == "dev" ? data.google_compute_network.dev_network.id : data.google_compute_network.prod_network.id + subnetwork_id = var.env == "dev" ? data.google_compute_subnetwork.dev_subnetwork.id : data.google_compute_subnetwork.prod_subnetwork.id project_id = var.project region = "us-central1" - service_name = module.leader.google_cloud_run_v2_service.name + service_name = "mpc-recovery-leader-${var.env}" } /* * Create multiple signer nodes @@ -138,7 +139,7 @@ module "signer" { zone = var.zone service_account_email = google_service_account.service_account.email docker_image = var.docker_image - connector_id = "projects/pagoda-shared-infrastructure/locations/us-central1/connectors/${var.env}-connector" + connector_id = var.env == "dev" ? var.dev-connector : var.prod-connector node_id = count.index @@ -165,7 +166,7 @@ module "leader" { zone = var.zone service_account_email = google_service_account.service_account.email docker_image = var.docker_image - connector_id = "projects/pagoda-shared-infrastructure/locations/us-central1/connectors/${var.env}-connector" + connector_id = var.env == "dev" ? var.dev-connector : var.prod-connector signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) near_rpc = local.workspace.near_rpc diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf index 2e5b284b6..29d7f6910 100644 --- a/infra/modules/leader/main.tf +++ b/infra/modules/leader/main.tf @@ -87,23 +87,6 @@ resource "google_cloud_run_v2_service" "leader" { } } } - - lifecycle { - # List of fields we don't want to see a diff for in terraform. Most of these fields are set - # by GCP and is metadata we don't want to account when considering changes in the service. - ignore_changes = [ - metadata[0].annotations["client.knative.dev/user-image"], - metadata[0].annotations["run.googleapis.com/client-name"], - metadata[0].annotations["run.googleapis.com/client-version"], - metadata[0].annotations["run.googleapis.com/launch-stage"], - metadata[0].annotations["run.googleapis.com/operation-id"], - template[0].metadata[0].annotations["client.knative.dev/user-image"], - template[0].metadata[0].annotations["run.googleapis.com/client-version"], - template[0].metadata[0].annotations["run.googleapis.com/client-name"], - template[0].metadata[0].labels["client.knative.dev/nonce"], - template[0].metadata[0].labels["run.googleapis.com/startupProbeType"], - ] - } } // Allow unauthenticated requests diff --git a/infra/modules/signer/main.tf b/infra/modules/signer/main.tf index 45c94ff74..d2f5dff9d 100644 --- a/infra/modules/signer/main.tf +++ b/infra/modules/signer/main.tf @@ -86,23 +86,6 @@ resource "google_cloud_run_v2_service" "signer" { } } } - - lifecycle { - # List of fields we don't want to see a diff for in terraform. Most of these fields are set - # by GCP and is metadata we don't want to account when considering changes in the service. - ignore_changes = [ - metadata[0].annotations["client.knative.dev/user-image"], - metadata[0].annotations["run.googleapis.com/client-name"], - metadata[0].annotations["run.googleapis.com/client-version"], - metadata[0].annotations["run.googleapis.com/launch-stage"], - metadata[0].annotations["run.googleapis.com/operation-id"], - template[0].metadata[0].annotations["client.knative.dev/user-image"], - template[0].metadata[0].annotations["run.googleapis.com/client-version"], - template[0].metadata[0].annotations["run.googleapis.com/client-name"], - template[0].metadata[0].labels["client.knative.dev/nonce"], - template[0].metadata[0].labels["run.googleapis.com/startupProbeType"], - ] - } } // Allow unauthenticated requests diff --git a/infra/terraform-dev.tfvars b/infra/terraform-dev.tfvars index 02f1a612e..8196fbe68 100644 --- a/infra/terraform-dev.tfvars +++ b/infra/terraform-dev.tfvars @@ -1,6 +1,6 @@ env = "dev" project = "pagoda-discovery-platform-dev" -docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery" +docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:f405e5594e666ef6a6e5ccf701c0d70c9673d4fe" account_creator_id = "mpc-recovery-dev-creator.testnet" account_creator_sk_secret_id = "mpc-account-creator-sk-dev" diff --git a/infra/variables.tf b/infra/variables.tf index 9978857f9..b0a419ae0 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -53,3 +53,33 @@ variable "signer_configs" { sk_share_secret_id = string })) } + +variable "dev-connector" { + default = "projects/pagoda-shared-infrastructure/locations/us-east1/connectors/dev-connector1" +} + +variable "prod-connector" { + default = "projects/pagoda-shared-infrastructure/locations/us-east1/connectors/prod-us-east1-connector" +} + +data "google_compute_subnetwork" "dev_subnetwork" { + name = "dev-us-central1" + project = "pagoda-shared-infrastructure" + region = "us-central1" +} + +data "google_compute_subnetwork" "prod_subnetwork" { + name = "prod-us-central1" + project = "pagoda-shared-infrastructure" + region = "us-central1" +} + +data "google_compute_network" "dev_network" { + name = "dev" + project = "pagoda-shared-infrastructure" +} + +data "google_compute_network" "prod_network" { + name = "prod" + project = "pagoda-shared-infrastructure" +} \ No newline at end of file From d8a369032bafafb4e457cc34274e144b6114874e Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Tue, 24 Oct 2023 15:23:11 -0600 Subject: [PATCH 04/20] separate prod and dev infra --- .github/workflows/deploy-dev.yml | 88 ++++++ .github/workflows/deploy-prod.yml | 141 ++++++++++ infra/modules/leader/main.tf | 3 +- infra/modules/leader/variables.tf | 4 + infra/modules/signer/main.tf | 2 +- infra/modules/signer/variables.tf | 4 + .../backend-config-dev.tfvars | 0 infra/{ => mpc-recovery-dev}/main.tf | 52 ++-- infra/{ => mpc-recovery-dev}/migration.py | 0 infra/{ => mpc-recovery-dev}/output.tf | 0 .../terraform-dev.tfvars | 0 infra/{ => mpc-recovery-dev}/variables.tf | 0 .../backend-config-prod.tfvars | 0 infra/mpc-recovery-prod/main.tf | 263 ++++++++++++++++++ infra/mpc-recovery-prod/migration.py | 22 ++ infra/mpc-recovery-prod/output.tf | 3 + infra/mpc-recovery-prod/terraform-dev.tfvars | 22 ++ infra/mpc-recovery-prod/variables.tf | 85 ++++++ 18 files changed, 662 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/deploy-dev.yml create mode 100644 .github/workflows/deploy-prod.yml rename infra/{ => mpc-recovery-dev}/backend-config-dev.tfvars (100%) rename infra/{ => mpc-recovery-dev}/main.tf (74%) rename infra/{ => mpc-recovery-dev}/migration.py (100%) rename infra/{ => mpc-recovery-dev}/output.tf (100%) rename infra/{ => mpc-recovery-dev}/terraform-dev.tfvars (100%) rename infra/{ => mpc-recovery-dev}/variables.tf (100%) rename infra/{ => mpc-recovery-prod}/backend-config-prod.tfvars (100%) create mode 100644 infra/mpc-recovery-prod/main.tf create mode 100644 infra/mpc-recovery-prod/migration.py create mode 100644 infra/mpc-recovery-prod/output.tf create mode 100644 infra/mpc-recovery-prod/terraform-dev.tfvars create mode 100644 infra/mpc-recovery-prod/variables.tf diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml new file mode 100644 index 000000000..ac18fe9f2 --- /dev/null +++ b/.github/workflows/deploy-dev.yml @@ -0,0 +1,88 @@ +name: Deploy to Dev environment. +on: + pull_request: + branches: + - develop + +env: + PROJECT_DEV: "pagoda-discovery-platform-dev" + REGION: "us-east1" + IMAGE: us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} + PR_NUMBER: ${{ github.event.number }} + +jobs: + build-mpc-recovery: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + name: "Checkout mpc-recovery" + + - name: Login to Artifact Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGION }}-docker.pkg.dev + username: _json_key + password: ${{ secrets.GCP_CREDENTIALS_DEV }} + + - name: Build Docker image and push to Google Artifact Registry + id: docker-push-tagged + uses: docker/build-push-action@v4 + with: + push: true + file: ./Dockerfile + tags: "${{ env.IMAGE }}:${{ github.sha }}" + + deploy-mpc-recovery-dev: + if: contains(github.event.pull_request.labels.*.name, 'deploy-test') + runs-on: ubuntu-latest + needs: build-mpc-recovery + env: + name: DEV + steps: + - uses: actions/checkout@v3 + name: "Checkout mpc-recovery" + + - name: "Authenticate to GCloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GCP_CREDENTIALS_DEV }}" + + - name: Deploy leader to Cloud Run + id: deploy-leader + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-leader-dev + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + - name: Deploy Signer to Cloud Run + id: deploy-signer-0 + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-0-dev + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + - name: Deploy Signer to Cloud Run + id: deploy-signer-1 + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-1-dev + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + - name: Deploy Signer to Cloud Run + id: deploy-signer-2 + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-2-dev + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" \ No newline at end of file diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml new file mode 100644 index 000000000..ac9a873b6 --- /dev/null +++ b/.github/workflows/deploy-prod.yml @@ -0,0 +1,141 @@ +name: Deploy to Prod environments. +on: + pull_request: + branches: + - main + +env: + PROJECT_DEV: "pagoda-discovery-platform-prod" + REGION: "us-east1" + IMAGE: us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery:${{ github.sha }} + PR_NUMBER: ${{ github.event.number }} + +jobs: + build-mpc-recovery: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + name: "Checkout mpc-recovery" + + - name: Login to Artifact Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGION }}-docker.pkg.dev + username: _json_key + password: ${{ secrets.GCP_CREDENTIALS_DEV }} + + - name: Build Docker image and push to Google Artifact Registry + id: docker-push-tagged + uses: docker/build-push-action@v4 + with: + push: true + file: ./Dockerfile + tags: "${{ env.IMAGE }}:${{ github.sha }}" + + deploy-mpc-recovery-testnet: + runs-on: ubuntu-latest + needs: build-mpc-recovery + env: + name: PROD + steps: + - uses: actions/checkout@v3 + name: "Checkout mpc-recovery" + + - name: "Authenticate to GCloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GCP_CREDENTIALS_DEV }}" + + - name: Deploy leader to Cloud Run Testnet + id: deploy-leader + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-leader-testnet + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + - name: Deploy signer to Cloud Run testnet + id: deploy-signer-0-testnet + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-0-dev + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + - name: Deploy signer to Cloud Run testnet + id: deploy-signer-1-testnet + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-1-dev + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + - name: Deploy signer to Cloud Run testnet + id: deploy-signer-2-testnet + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-2-dev + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + deploy-mpc-recovery-mainnet: + runs-on: ubuntu-latest + needs: build-mpc-recovery + env: + name: PROD + steps: + - uses: actions/checkout@v3 + name: "Checkout mpc-recovery" + + - name: "Authenticate to GCloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GCP_CREDENTIALS_DEV }}" + + - name: Deploy leader to Cloud Run mainnet + id: deploy-leader + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-leader-mainnet + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + - name: Deploy signer to Cloud Run mainnet + id: deploy-signer-0 + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-0-mainnet + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + - name: Deploy signer to Cloud Run mainnet + id: deploy-signer-1 + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-1-mainnet + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" + + - name: Deploy signer to Cloud Run mainnet + id: deploy-signer-2 + uses: google-github-actions/deploy-cloudrun@v1 + with: + image: "${{ env.IMAGE }}:${{ github.sha }}" + service: mpc-recovery-signer-2-mainnet + region: us-east1 + project_id: "${{ env.PROJECT_DEV }}" + tag: "pr-${{ github.event.number }}" \ No newline at end of file diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf index 29d7f6910..a3eadf848 100644 --- a/infra/modules/leader/main.tf +++ b/infra/modules/leader/main.tf @@ -1,5 +1,5 @@ resource "google_cloud_run_v2_service" "leader" { - name = "mpc-recovery-leader-${var.env}" + name = var.service_name location = var.region ingress = "INGRESS_TRAFFIC_ALL" @@ -76,7 +76,6 @@ resource "google_cloud_run_v2_service" "leader" { ports { container_port = 3000 } - resources { cpu_idle = false diff --git a/infra/modules/leader/variables.tf b/infra/modules/leader/variables.tf index 351750082..06aea606e 100644 --- a/infra/modules/leader/variables.tf +++ b/infra/modules/leader/variables.tf @@ -57,3 +57,7 @@ variable "account_creator_sk_secret_id" { variable "fast_auth_partners_secret_id" { type = string } + +variable "service_name" { + type = string +} \ No newline at end of file diff --git a/infra/modules/signer/main.tf b/infra/modules/signer/main.tf index d2f5dff9d..d354450e8 100644 --- a/infra/modules/signer/main.tf +++ b/infra/modules/signer/main.tf @@ -1,5 +1,5 @@ resource "google_cloud_run_v2_service" "signer" { - name = "mpc-recovery-signer-${var.node_id}-${var.env}" + name = var.service_name location = var.region ingress = "INGRESS_TRAFFIC_ALL" diff --git a/infra/modules/signer/variables.tf b/infra/modules/signer/variables.tf index a1d5aaf24..5eaab18a0 100644 --- a/infra/modules/signer/variables.tf +++ b/infra/modules/signer/variables.tf @@ -42,3 +42,7 @@ variable "sk_share_secret_id" { variable "oidc_providers_secret_id" { type = string } + +variable "service_name" { + type = string +} \ No newline at end of file diff --git a/infra/backend-config-dev.tfvars b/infra/mpc-recovery-dev/backend-config-dev.tfvars similarity index 100% rename from infra/backend-config-dev.tfvars rename to infra/mpc-recovery-dev/backend-config-dev.tfvars diff --git a/infra/main.tf b/infra/mpc-recovery-dev/main.tf similarity index 74% rename from infra/main.tf rename to infra/mpc-recovery-dev/main.tf index f7438c315..9ab216b6d 100644 --- a/infra/main.tf +++ b/infra/mpc-recovery-dev/main.tf @@ -13,9 +13,9 @@ terraform { } locals { - credentials = var.credentials != null ? var.credentials : file(var.credentials_file) - client_email = jsondecode(local.credentials).client_email - client_id = jsondecode(local.credentials).client_id + # credentials = var.credentials != null ? var.credentials : file(var.credentials_file) + # client_email = jsondecode(local.credentials).client_email + # client_id = jsondecode(local.credentials).client_id env = { defaults = { @@ -34,11 +34,12 @@ locals { } data "external" "git_checkout" { - program = ["${path.module}/scripts/get_sha.sh"] + program = ["${path.module}/../scripts/get_sha.sh"] } provider "google" { - credentials = local.credentials + # credentials = local.credentials + credentials = file("~/.config/gcloud/application_default_credentials.json") project = var.project region = var.region @@ -49,8 +50,8 @@ provider "google" { * Create brand new service account with basic IAM */ resource "google_service_account" "service_account" { - account_id = "mpc-recovery-${var.env}" - display_name = "MPC Recovery ${var.env} Account" + account_id = "mpc-recovery-dev" + display_name = "MPC Recovery dev Account" } resource "google_service_account_iam_binding" "serivce-account-iam" { @@ -58,7 +59,8 @@ resource "google_service_account_iam_binding" "serivce-account-iam" { role = "roles/iam.serviceAccountUser" members = [ - "serviceAccount:${local.client_email}", + # "serviceAccount:${local.client_email}", + "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" ] } @@ -108,38 +110,39 @@ resource "google_secret_manager_secret_iam_member" "fast_auth_partners_secret_ac module "mpc-signer-lb" { count = length(var.signer_configs) - source = "./modules/internal_cloudrun_lb" - name = "mpc-${var.env}-signer-${count.index}" - network_id = var.env == "dev" ? data.google_compute_network.dev_network.id : data.google_compute_network.prod_network.id - subnetwork_id = var.env == "dev" ? data.google_compute_subnetwork.dev_subnetwork.id : data.google_compute_subnetwork.prod_subnetwork.id + source = "../modules/internal_cloudrun_lb" + name = "mpc-dev-signer-${count.index}" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id project_id = var.project region = "us-central1" - service_name = "mpc-recovery-signer-${count.index}-${var.env}" + service_name = "mpc-recovery-signer-${count.index}-dev" } module "mpc-leader-lb" { - source = "./modules/internal_cloudrun_lb" - name = "mpc-${var.env}-leader" - network_id = var.env == "dev" ? data.google_compute_network.dev_network.id : data.google_compute_network.prod_network.id - subnetwork_id = var.env == "dev" ? data.google_compute_subnetwork.dev_subnetwork.id : data.google_compute_subnetwork.prod_subnetwork.id + source = "../modules/internal_cloudrun_lb" + name = "mpc-dev-leader" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id project_id = var.project region = "us-central1" - service_name = "mpc-recovery-leader-${var.env}" + service_name = "mpc-recovery-leader-dev" } /* * Create multiple signer nodes */ module "signer" { count = length(var.signer_configs) - source = "./modules/signer" + source = "../modules/signer" - env = var.env + env = "dev" + service_name = "mpc-recovery-signer-${count.index}-dev" project = var.project region = var.region zone = var.zone service_account_email = google_service_account.service_account.email docker_image = var.docker_image - connector_id = var.env == "dev" ? var.dev-connector : var.prod-connector + connector_id = var.prod-connector node_id = count.index @@ -158,15 +161,16 @@ module "signer" { * Create leader node */ module "leader" { - source = "./modules/leader" + source = "../modules/leader" - env = var.env + env = "dev" + service_name = "mpc-recovery-leader-dev" project = var.project region = var.region zone = var.zone service_account_email = google_service_account.service_account.email docker_image = var.docker_image - connector_id = var.env == "dev" ? var.dev-connector : var.prod-connector + connector_id = var.prod-connector signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) near_rpc = local.workspace.near_rpc diff --git a/infra/migration.py b/infra/mpc-recovery-dev/migration.py similarity index 100% rename from infra/migration.py rename to infra/mpc-recovery-dev/migration.py diff --git a/infra/output.tf b/infra/mpc-recovery-dev/output.tf similarity index 100% rename from infra/output.tf rename to infra/mpc-recovery-dev/output.tf diff --git a/infra/terraform-dev.tfvars b/infra/mpc-recovery-dev/terraform-dev.tfvars similarity index 100% rename from infra/terraform-dev.tfvars rename to infra/mpc-recovery-dev/terraform-dev.tfvars diff --git a/infra/variables.tf b/infra/mpc-recovery-dev/variables.tf similarity index 100% rename from infra/variables.tf rename to infra/mpc-recovery-dev/variables.tf diff --git a/infra/backend-config-prod.tfvars b/infra/mpc-recovery-prod/backend-config-prod.tfvars similarity index 100% rename from infra/backend-config-prod.tfvars rename to infra/mpc-recovery-prod/backend-config-prod.tfvars diff --git a/infra/mpc-recovery-prod/main.tf b/infra/mpc-recovery-prod/main.tf new file mode 100644 index 000000000..e51feb00c --- /dev/null +++ b/infra/mpc-recovery-prod/main.tf @@ -0,0 +1,263 @@ +terraform { + backend "gcs" { + bucket = "mpc-recovery-terraform-prod" + prefix = "state/mpc-recovery" + } + + required_providers { + google = { + source = "hashicorp/google" + version = "4.73.0" + } + } +} + +locals { + credentials = var.credentials != null ? var.credentials : file(var.credentials_file) + client_email = jsondecode(local.credentials).client_email + client_id = jsondecode(local.credentials).client_id + + env = { + defaults = { + near_rpc = "https://rpc.testnet.near.org" + near_root_account = "testnet" + } + testnet = { + } + mainnet = { + near_rpc = "https://rpc.mainnet.near.org" + near_root_account = "near" + } + } + + workspace = merge(local.env["defaults"], contains(keys(local.env), terraform.workspace) ? local.env[terraform.workspace] : local.env["defaults"]) +} + +data "external" "git_checkout" { + program = ["${path.module}/../scripts/get_sha.sh"] +} + +provider "google" { + credentials = local.credentials + # credentials = file("~/.config/gcloud/application_default_credentials.json") + + project = var.project + region = var.region + zone = var.zone +} + +/* + * Create brand new service account with basic IAM + */ +resource "google_service_account" "service_account" { + account_id = "mpc-recovery-prod" + display_name = "MPC Recovery prod Account" +} + +resource "google_service_account_iam_binding" "serivce-account-iam" { + service_account_id = google_service_account.service_account.name + role = "roles/iam.serviceAccountUser" + + members = [ + "serviceAccount:${local.client_email}", + # "serviceAccount:mpc-recovery@pagoda-discovery-platform-prod.iam.gserviceaccount.com" + ] +} + +resource "google_project_iam_member" "service-account-datastore-user" { + project = var.project + role = "roles/datastore.user" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +/* + * Ensure service account has access to Secret Manager variables + */ +resource "google_secret_manager_secret_iam_member" "cipher_key_secret_access" { + count = length(var.signer_configs) + + secret_id = var.signer_configs[count.index].cipher_key_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "secret_share_secret_access" { + count = length(var.signer_configs) + + secret_id = var.signer_configs[count.index].sk_share_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "oidc_providers_secret_access" { + secret_id = var.oidc_providers_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "account_creator_secret_access" { + secret_id = var.account_creator_sk_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +resource "google_secret_manager_secret_iam_member" "fast_auth_partners_secret_access" { + secret_id = var.fast_auth_partners_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +module "mpc-signer-lb-mainnet" { + + count = length(var.signer_configs) + source = "../modules/internal_cloudrun_lb" + name = "mpc-prod-signer-${count.index}-mainnet" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-signer-${count.index}-mainnet" +} + +module "mpc-signer-lb-testnet" { + + count = length(var.signer_configs) + source = "../modules/internal_cloudrun_lb" + name = "mpc-prod-signer-${count.index}-testnet" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-signer-${count.index}-testnet" +} + +module "mpc-leader-lb-mainnet" { + source = "../modules/internal_cloudrun_lb" + name = "mpc-prod-leader-mainnet" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-leader-mainnet" +} + +module "mpc-leader-lb-testnet" { + source = "../modules/internal_cloudrun_lb" + name = "mpc-prod-leader-testnet" + network_id = data.google_compute_network.prod_network.id + subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id + project_id = var.project + region = "us-central1" + service_name = "mpc-recovery-leader-testnet" +} +/* + * Create multiple signer nodes + */ +module "signer-mainnet" { + count = length(var.signer_configs) + source = "../modules/signer" + + env = "prod" + service_name = "mpc-recovery-signer-${count.index}-mainnet" + project = var.project + region = var.region + zone = var.zone + service_account_email = google_service_account.service_account.email + docker_image = var.docker_image + connector_id = var.prod-connector + + node_id = count.index + + oidc_providers_secret_id = var.oidc_providers_secret_id + cipher_key_secret_id = var.signer_configs[count.index].cipher_key_secret_id + sk_share_secret_id = var.signer_configs[count.index].sk_share_secret_id + + depends_on = [ + google_secret_manager_secret_iam_member.cipher_key_secret_access, + google_secret_manager_secret_iam_member.secret_share_secret_access, + google_secret_manager_secret_iam_member.oidc_providers_secret_access + ] +} + +module "signer-testnet" { + count = length(var.signer_configs) + source = "../modules/signer" + + env = "prod" + service_name = "mpc-recovery-signer-${count.index}-testnet" + project = var.project + region = var.region + zone = var.zone + service_account_email = google_service_account.service_account.email + docker_image = var.docker_image + connector_id = var.prod-connector + + node_id = count.index + + oidc_providers_secret_id = var.oidc_providers_secret_id + cipher_key_secret_id = var.signer_configs[count.index].cipher_key_secret_id + sk_share_secret_id = var.signer_configs[count.index].sk_share_secret_id + + depends_on = [ + google_secret_manager_secret_iam_member.cipher_key_secret_access, + google_secret_manager_secret_iam_member.secret_share_secret_access, + google_secret_manager_secret_iam_member.oidc_providers_secret_access + ] +} + +/* + * Create leader node + */ +module "leader-mainnet" { + source = "../modules/leader" + + env = "prod" + service_name = "mpc-recovery-leader-mainnet" + project = var.project + region = var.region + zone = var.zone + service_account_email = google_service_account.service_account.email + docker_image = var.docker_image + connector_id = var.prod-connector + + signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) + near_rpc = local.workspace.near_rpc + near_root_account = local.workspace.near_root_account + account_creator_id = var.account_creator_id + + account_creator_sk_secret_id = var.account_creator_sk_secret_id + fast_auth_partners_secret_id = var.fast_auth_partners_secret_id + + depends_on = [ + google_secret_manager_secret_iam_member.account_creator_secret_access, + google_secret_manager_secret_iam_member.fast_auth_partners_secret_access, + module.signer + ] +} + +module "leader-testnet" { + source = "../modules/leader" + + env = "prod" + service_name = "mpc-recovery-leader-testnet" + project = var.project + region = var.region + zone = var.zone + service_account_email = google_service_account.service_account.email + docker_image = var.docker_image + connector_id = var.prod-connector + + signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) + near_rpc = local.workspace.near_rpc + near_root_account = local.workspace.near_root_account + account_creator_id = var.account_creator_id + + account_creator_sk_secret_id = var.account_creator_sk_secret_id + fast_auth_partners_secret_id = var.fast_auth_partners_secret_id + + depends_on = [ + google_secret_manager_secret_iam_member.account_creator_secret_access, + google_secret_manager_secret_iam_member.fast_auth_partners_secret_access, + module.signer + ] +} diff --git a/infra/mpc-recovery-prod/migration.py b/infra/mpc-recovery-prod/migration.py new file mode 100644 index 000000000..afbdc3d2b --- /dev/null +++ b/infra/mpc-recovery-prod/migration.py @@ -0,0 +1,22 @@ +from google.oauth2 import service_account +from google.cloud import datastore + +credentials_source = service_account.Credentials.from_service_account_file( + '../source-service-keys.json') +client_source = datastore.Client(project="pagoda-discovery-platform-dev", credentials=credentials_source) + +credentials_target = service_account.Credentials.from_service_account_file( + '../target-service-keys.json') +client_target = datastore.Client(project="pagoda-discovery-platform-prod", credentials=credentials_target) + +print('Fetching source entities') +query = credentials_source.query(kind="EncryptedUserCredentials-dev") +entities = [] +for entity in list(query.fetch()): + entity.key = client_target.key('EncryptedUserCredentials-mainnet').completed_key(entity.key.id_or_name) + print(entity.key) + print(entity) + entities.append(entity) + +print("Uploading a total of " + str(len(entities)) + " entities to target") +client_target.put_multi(entities) \ No newline at end of file diff --git a/infra/mpc-recovery-prod/output.tf b/infra/mpc-recovery-prod/output.tf new file mode 100644 index 000000000..3447956d7 --- /dev/null +++ b/infra/mpc-recovery-prod/output.tf @@ -0,0 +1,3 @@ +output "leader_node" { + value = module.leader.node.uri +} diff --git a/infra/mpc-recovery-prod/terraform-dev.tfvars b/infra/mpc-recovery-prod/terraform-dev.tfvars new file mode 100644 index 000000000..8196fbe68 --- /dev/null +++ b/infra/mpc-recovery-prod/terraform-dev.tfvars @@ -0,0 +1,22 @@ +env = "dev" +project = "pagoda-discovery-platform-dev" +docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:f405e5594e666ef6a6e5ccf701c0d70c9673d4fe" + +account_creator_id = "mpc-recovery-dev-creator.testnet" +account_creator_sk_secret_id = "mpc-account-creator-sk-dev" +oidc_providers_secret_id = "mpc-allowed-oidc-providers-dev" +fast_auth_partners_secret_id = "mpc-fast-auth-partners-dev" +signer_configs = [ + { + cipher_key_secret_id = "mpc-cipher-0-dev" + sk_share_secret_id = "mpc-sk-share-0-dev" + }, + { + cipher_key_secret_id = "mpc-cipher-1-dev" + sk_share_secret_id = "mpc-sk-share-1-dev" + }, + { + cipher_key_secret_id = "mpc-cipher-2-dev" + sk_share_secret_id = "mpc-sk-share-2-dev" + } +] diff --git a/infra/mpc-recovery-prod/variables.tf b/infra/mpc-recovery-prod/variables.tf new file mode 100644 index 000000000..b0a419ae0 --- /dev/null +++ b/infra/mpc-recovery-prod/variables.tf @@ -0,0 +1,85 @@ +variable "env" { +} + +variable "project" { +} + +variable "credentials_file" { + default = null +} + +variable "credentials" { + default = null +} + +variable "region" { + default = "us-east1" +} + +variable "zone" { + default = "us-east1-c" +} + +variable "docker_image" { + type = string +} + +# Application variables +variable "account_creator_id" { + default = "tmp_acount_creator.serhii.testnet" +} + +variable "external_signer_node_urls" { + type = list(string) + default = [] +} + +# Secrets +variable "account_creator_sk_secret_id" { + type = string +} + +variable "oidc_providers_secret_id" { + type = string +} + +variable "fast_auth_partners_secret_id" { + type = string +} + +variable "signer_configs" { + type = list(object({ + cipher_key_secret_id = string + sk_share_secret_id = string + })) +} + +variable "dev-connector" { + default = "projects/pagoda-shared-infrastructure/locations/us-east1/connectors/dev-connector1" +} + +variable "prod-connector" { + default = "projects/pagoda-shared-infrastructure/locations/us-east1/connectors/prod-us-east1-connector" +} + +data "google_compute_subnetwork" "dev_subnetwork" { + name = "dev-us-central1" + project = "pagoda-shared-infrastructure" + region = "us-central1" +} + +data "google_compute_subnetwork" "prod_subnetwork" { + name = "prod-us-central1" + project = "pagoda-shared-infrastructure" + region = "us-central1" +} + +data "google_compute_network" "dev_network" { + name = "dev" + project = "pagoda-shared-infrastructure" +} + +data "google_compute_network" "prod_network" { + name = "prod" + project = "pagoda-shared-infrastructure" +} \ No newline at end of file From 11354c2c5ab3e0526b383649a294050f1fc6bc1b Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Wed, 1 Nov 2023 08:14:22 -0600 Subject: [PATCH 05/20] changed connector id for dev --- infra/modules/leader/main.tf | 2 +- infra/modules/signer/main.tf | 4 ++-- infra/mpc-recovery-dev/main.tf | 16 ++++++++-------- infra/partner/main.tf | 4 ++++ 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf index b3e24ea8d..0c506fb6b 100644 --- a/infra/modules/leader/main.tf +++ b/infra/modules/leader/main.tf @@ -10,7 +10,7 @@ resource "google_cloud_run_v2_service" "leader" { vpc_access { connector = var.connector_id - egress = "ALL_TRAFFIC" + egress = "PRIVATE_RANGES_ONLY" } scaling { diff --git a/infra/modules/signer/main.tf b/infra/modules/signer/main.tf index fced77c38..bfa61cd57 100644 --- a/infra/modules/signer/main.tf +++ b/infra/modules/signer/main.tf @@ -9,8 +9,8 @@ resource "google_cloud_run_v2_service" "signer" { annotations = var.metadata_annotations == null ? null : var.metadata_annotations vpc_access { - connector = var.connector_id - egress = "ALL_TRAFFIC" + connector = var.connector_id == null ? null : var.connector_id + egress = "PRIVATE_RANGES_ONLY" } scaling { diff --git a/infra/mpc-recovery-dev/main.tf b/infra/mpc-recovery-dev/main.tf index 4dd4306a9..f498a9740 100644 --- a/infra/mpc-recovery-dev/main.tf +++ b/infra/mpc-recovery-dev/main.tf @@ -13,9 +13,9 @@ terraform { } locals { - # credentials = var.credentials != null ? var.credentials : file(var.credentials_file) - # client_email = jsondecode(local.credentials).client_email - # client_id = jsondecode(local.credentials).client_id + credentials = var.credentials != null ? var.credentials : file(var.credentials_file) + client_email = jsondecode(local.credentials).client_email + client_id = jsondecode(local.credentials).client_id env = { defaults = { @@ -38,8 +38,8 @@ data "external" "git_checkout" { } provider "google" { - # credentials = local.credentials - credentials = file("~/.config/gcloud/application_default_credentials.json") + credentials = local.credentials + # credentials = file("~/.config/gcloud/application_default_credentials.json") project = var.project region = var.region @@ -59,8 +59,8 @@ resource "google_service_account_iam_binding" "serivce-account-iam" { role = "roles/iam.serviceAccountUser" members = [ - # "serviceAccount:${local.client_email}", - "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" + "serviceAccount:${local.client_email}", + # "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" ] } @@ -142,7 +142,7 @@ module "signer" { zone = var.zone service_account_email = google_service_account.service_account.email docker_image = var.docker_image - connector_id = var.prod-connector + connector_id = var.dev-connector node_id = count.index diff --git a/infra/partner/main.tf b/infra/partner/main.tf index 4f366585d..0a7dd79a5 100644 --- a/infra/partner/main.tf +++ b/infra/partner/main.tf @@ -75,6 +75,7 @@ module "signer" { source = "../modules/signer" env = var.env + service_name = "partner-service-name" project = var.project region = var.region zone = var.zone @@ -87,6 +88,9 @@ module "signer" { sk_share_secret_id = var.sk_share_secret_id oidc_providers_secret_id = var.oidc_providers_secret_id + # optional + connector_id = "partner-vpc-connector-id" + jwt_signature_pk_url = var.jwt_signature_pk_url depends_on = [ From 7ff39aa0caa163f6299fc286016ffebf0d3b4634 Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Wed, 1 Nov 2023 08:39:55 -0600 Subject: [PATCH 06/20] removed dev workflow, updated prod workflow --- .github/workflows/deploy-dev.yml | 88 ------------------------------- .github/workflows/deploy-prod.yml | 44 +++++++--------- 2 files changed, 20 insertions(+), 112 deletions(-) delete mode 100644 .github/workflows/deploy-dev.yml diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml deleted file mode 100644 index ac18fe9f2..000000000 --- a/.github/workflows/deploy-dev.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Deploy to Dev environment. -on: - pull_request: - branches: - - develop - -env: - PROJECT_DEV: "pagoda-discovery-platform-dev" - REGION: "us-east1" - IMAGE: us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} - PR_NUMBER: ${{ github.event.number }} - -jobs: - build-mpc-recovery: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - name: "Checkout mpc-recovery" - - - name: Login to Artifact Registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGION }}-docker.pkg.dev - username: _json_key - password: ${{ secrets.GCP_CREDENTIALS_DEV }} - - - name: Build Docker image and push to Google Artifact Registry - id: docker-push-tagged - uses: docker/build-push-action@v4 - with: - push: true - file: ./Dockerfile - tags: "${{ env.IMAGE }}:${{ github.sha }}" - - deploy-mpc-recovery-dev: - if: contains(github.event.pull_request.labels.*.name, 'deploy-test') - runs-on: ubuntu-latest - needs: build-mpc-recovery - env: - name: DEV - steps: - - uses: actions/checkout@v3 - name: "Checkout mpc-recovery" - - - name: "Authenticate to GCloud" - uses: "google-github-actions/auth@v1" - with: - credentials_json: "${{ secrets.GCP_CREDENTIALS_DEV }}" - - - name: Deploy leader to Cloud Run - id: deploy-leader - uses: google-github-actions/deploy-cloudrun@v1 - with: - image: "${{ env.IMAGE }}:${{ github.sha }}" - service: mpc-recovery-leader-dev - region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" - - - name: Deploy Signer to Cloud Run - id: deploy-signer-0 - uses: google-github-actions/deploy-cloudrun@v1 - with: - image: "${{ env.IMAGE }}:${{ github.sha }}" - service: mpc-recovery-signer-0-dev - region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" - - - name: Deploy Signer to Cloud Run - id: deploy-signer-1 - uses: google-github-actions/deploy-cloudrun@v1 - with: - image: "${{ env.IMAGE }}:${{ github.sha }}" - service: mpc-recovery-signer-1-dev - region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" - - - name: Deploy Signer to Cloud Run - id: deploy-signer-2 - uses: google-github-actions/deploy-cloudrun@v1 - with: - image: "${{ env.IMAGE }}:${{ github.sha }}" - service: mpc-recovery-signer-2-dev - region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" \ No newline at end of file diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index ac9a873b6..576a4c89a 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -1,14 +1,16 @@ name: Deploy to Prod environments. on: - pull_request: - branches: - - main + workflow_dispatch: + inputs: + network: + description: mainnet or testnet + required: true + env: - PROJECT_DEV: "pagoda-discovery-platform-prod" + PROJECT_PROD: "pagoda-discovery-platform-prod" REGION: "us-east1" IMAGE: us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery:${{ github.sha }} - PR_NUMBER: ${{ github.event.number }} jobs: build-mpc-recovery: @@ -34,6 +36,7 @@ jobs: deploy-mpc-recovery-testnet: runs-on: ubuntu-latest + if: github.event.inputs.network == 'testnet' needs: build-mpc-recovery env: name: PROD @@ -53,41 +56,38 @@ jobs: image: "${{ env.IMAGE }}:${{ github.sha }}" service: mpc-recovery-leader-testnet region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" + project_id: "${{ env.PROJECT_PROD }}" - name: Deploy signer to Cloud Run testnet id: deploy-signer-0-testnet uses: google-github-actions/deploy-cloudrun@v1 with: image: "${{ env.IMAGE }}:${{ github.sha }}" - service: mpc-recovery-signer-0-dev + service: mpc-recovery-signer-0-testnet region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" + project_id: "${{ env.PROJECT_PROD }}" - name: Deploy signer to Cloud Run testnet id: deploy-signer-1-testnet uses: google-github-actions/deploy-cloudrun@v1 with: image: "${{ env.IMAGE }}:${{ github.sha }}" - service: mpc-recovery-signer-1-dev + service: mpc-recovery-signer-1-testnet region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" + project_id: "${{ env.PROJECT_PROD }}" - name: Deploy signer to Cloud Run testnet id: deploy-signer-2-testnet uses: google-github-actions/deploy-cloudrun@v1 with: image: "${{ env.IMAGE }}:${{ github.sha }}" - service: mpc-recovery-signer-2-dev + service: mpc-recovery-signer-2-testnet region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" + project_id: "${{ env.PROJECT_PROD }}" deploy-mpc-recovery-mainnet: runs-on: ubuntu-latest + if: github.event.inputs.network == 'mainnet' needs: build-mpc-recovery env: name: PROD @@ -107,8 +107,7 @@ jobs: image: "${{ env.IMAGE }}:${{ github.sha }}" service: mpc-recovery-leader-mainnet region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" + project_id: "${{ env.PROJECT_PROD }}" - name: Deploy signer to Cloud Run mainnet id: deploy-signer-0 @@ -117,8 +116,7 @@ jobs: image: "${{ env.IMAGE }}:${{ github.sha }}" service: mpc-recovery-signer-0-mainnet region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" + project_id: "${{ env.PROJECT_PROD }}" - name: Deploy signer to Cloud Run mainnet id: deploy-signer-1 @@ -127,8 +125,7 @@ jobs: image: "${{ env.IMAGE }}:${{ github.sha }}" service: mpc-recovery-signer-1-mainnet region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" + project_id: "${{ env.PROJECT_PROD }}" - name: Deploy signer to Cloud Run mainnet id: deploy-signer-2 @@ -137,5 +134,4 @@ jobs: image: "${{ env.IMAGE }}:${{ github.sha }}" service: mpc-recovery-signer-2-mainnet region: us-east1 - project_id: "${{ env.PROJECT_DEV }}" - tag: "pr-${{ github.event.number }}" \ No newline at end of file + project_id: "${{ env.PROJECT_PROD }}" \ No newline at end of file From 64e8e5d8efe6f0222ae84155244868b1fc2fb33e Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Wed, 1 Nov 2023 08:46:27 -0600 Subject: [PATCH 07/20] reflected dev and prod env --- infra/mpc-recovery-prod/main.tf | 5 +++++ infra/mpc-recovery-prod/variables.tf | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/infra/mpc-recovery-prod/main.tf b/infra/mpc-recovery-prod/main.tf index e51feb00c..948d5d5a1 100644 --- a/infra/mpc-recovery-prod/main.tf +++ b/infra/mpc-recovery-prod/main.tf @@ -165,6 +165,7 @@ module "signer-mainnet" { service_account_email = google_service_account.service_account.email docker_image = var.docker_image connector_id = var.prod-connector + jwt_signature_pk_url = var.jwt_signature_pk_url node_id = count.index @@ -191,6 +192,7 @@ module "signer-testnet" { service_account_email = google_service_account.service_account.email docker_image = var.docker_image connector_id = var.prod-connector + jwt_signature_pk_url = var.jwt_signature_pk_url node_id = count.index @@ -219,6 +221,7 @@ module "leader-mainnet" { service_account_email = google_service_account.service_account.email docker_image = var.docker_image connector_id = var.prod-connector + jwt_signature_pk_url = var.jwt_signature_pk_url signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) near_rpc = local.workspace.near_rpc @@ -246,12 +249,14 @@ module "leader-testnet" { service_account_email = google_service_account.service_account.email docker_image = var.docker_image connector_id = var.prod-connector + jwt_signature_pk_url = var.jwt_signature_pk_url signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) near_rpc = local.workspace.near_rpc near_root_account = local.workspace.near_root_account account_creator_id = var.account_creator_id + account_creator_sk_secret_id = var.account_creator_sk_secret_id fast_auth_partners_secret_id = var.fast_auth_partners_secret_id diff --git a/infra/mpc-recovery-prod/variables.tf b/infra/mpc-recovery-prod/variables.tf index b0a419ae0..46ee89ab5 100644 --- a/infra/mpc-recovery-prod/variables.tf +++ b/infra/mpc-recovery-prod/variables.tf @@ -82,4 +82,8 @@ data "google_compute_network" "dev_network" { data "google_compute_network" "prod_network" { name = "prod" project = "pagoda-shared-infrastructure" +} + +variable "jwt_signature_pk_url" { + type = string } \ No newline at end of file From 321a1d800e736bd9fd7f7461477c4a8f26d2f82b Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Wed, 1 Nov 2023 09:03:19 -0600 Subject: [PATCH 08/20] updated directory of dev workflow --- .github/workflows/terraform-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terraform-dev.yml b/.github/workflows/terraform-dev.yml index 5fdc9c98d..1aea0ba54 100644 --- a/.github/workflows/terraform-dev.yml +++ b/.github/workflows/terraform-dev.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write defaults: run: - working-directory: ./infra + working-directory: ./infra/mpc-recovery-dev steps: - name: Checkout uses: actions/checkout@v3 From 554837827cfd03b340a43d5b2573ccf5455e4844 Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Wed, 1 Nov 2023 09:04:27 -0600 Subject: [PATCH 09/20] updated all workflow dev directories --- .github/workflows/terraform-feature-env-destroy.yml | 2 +- .github/workflows/terraform-feature-env.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/terraform-feature-env-destroy.yml b/.github/workflows/terraform-feature-env-destroy.yml index d59626cae..65723ad16 100644 --- a/.github/workflows/terraform-feature-env-destroy.yml +++ b/.github/workflows/terraform-feature-env-destroy.yml @@ -14,7 +14,7 @@ jobs: pull-requests: write defaults: run: - working-directory: ./infra + working-directory: ./infra/mpc-recovery-dev env: PR_NUMBER: ${{ github.event.number }} steps: diff --git a/.github/workflows/terraform-feature-env.yml b/.github/workflows/terraform-feature-env.yml index 6b1d4ba5e..35c8c7efa 100644 --- a/.github/workflows/terraform-feature-env.yml +++ b/.github/workflows/terraform-feature-env.yml @@ -15,7 +15,7 @@ jobs: checks: read defaults: run: - working-directory: ./infra + working-directory: ./infra/mpc-recovery-dev env: PR_NUMBER: ${{ github.event.number }} steps: From a8a6a16cc6b7a6b8ece938985ba799acb47768ac Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Wed, 1 Nov 2023 09:34:39 -0600 Subject: [PATCH 10/20] removed odic variable --- infra/modules/signer/variables.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/infra/modules/signer/variables.tf b/infra/modules/signer/variables.tf index f16246870..e78de7762 100644 --- a/infra/modules/signer/variables.tf +++ b/infra/modules/signer/variables.tf @@ -39,10 +39,6 @@ variable "sk_share_secret_id" { type = string } -variable "oidc_providers_secret_id" { - type = string -} - variable "service_name" { type = string } From 7115c9b5eff34c13b4db51a3a5bcb1683abe3fb9 Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Wed, 1 Nov 2023 12:57:36 -0600 Subject: [PATCH 11/20] updated prod workflow to have selectable networks --- .github/workflows/deploy-prod.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 576a4c89a..4a55d650c 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -3,7 +3,14 @@ on: workflow_dispatch: inputs: network: - description: mainnet or testnet + type: choice + options: + - mainnet + - testnet + description: mainnet or testnet network + required: true + version: + description: What mainnet version number is this deployment? (e.g. v0.1.0) required: true @@ -57,6 +64,7 @@ jobs: service: mpc-recovery-leader-testnet region: us-east1 project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" - name: Deploy signer to Cloud Run testnet id: deploy-signer-0-testnet @@ -66,6 +74,7 @@ jobs: service: mpc-recovery-signer-0-testnet region: us-east1 project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" - name: Deploy signer to Cloud Run testnet id: deploy-signer-1-testnet @@ -75,6 +84,7 @@ jobs: service: mpc-recovery-signer-1-testnet region: us-east1 project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" - name: Deploy signer to Cloud Run testnet id: deploy-signer-2-testnet @@ -84,6 +94,7 @@ jobs: service: mpc-recovery-signer-2-testnet region: us-east1 project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" deploy-mpc-recovery-mainnet: runs-on: ubuntu-latest @@ -108,6 +119,7 @@ jobs: service: mpc-recovery-leader-mainnet region: us-east1 project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" - name: Deploy signer to Cloud Run mainnet id: deploy-signer-0 @@ -117,6 +129,7 @@ jobs: service: mpc-recovery-signer-0-mainnet region: us-east1 project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" - name: Deploy signer to Cloud Run mainnet id: deploy-signer-1 @@ -126,6 +139,7 @@ jobs: service: mpc-recovery-signer-1-mainnet region: us-east1 project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" - name: Deploy signer to Cloud Run mainnet id: deploy-signer-2 @@ -134,4 +148,5 @@ jobs: image: "${{ env.IMAGE }}:${{ github.sha }}" service: mpc-recovery-signer-2-mainnet region: us-east1 - project_id: "${{ env.PROJECT_PROD }}" \ No newline at end of file + project_id: "${{ env.PROJECT_PROD }}" + tag: "${{ github.event.inputs.version }}" \ No newline at end of file From ddc2222933920028722edfb83b3156e296890e35 Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Tue, 7 Nov 2023 09:09:36 -0700 Subject: [PATCH 12/20] added var.env back to fix pipeline --- infra/mpc-recovery-dev/main.tf | 30 ++++++++++---------- infra/mpc-recovery-prod/main.tf | 10 +++---- infra/mpc-recovery-prod/terraform-dev.tfvars | 24 ++++++++-------- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/infra/mpc-recovery-dev/main.tf b/infra/mpc-recovery-dev/main.tf index 3797084c4..1289d33bd 100644 --- a/infra/mpc-recovery-dev/main.tf +++ b/infra/mpc-recovery-dev/main.tf @@ -13,9 +13,9 @@ terraform { } locals { - credentials = var.credentials != null ? var.credentials : file(var.credentials_file) - client_email = jsondecode(local.credentials).client_email - client_id = jsondecode(local.credentials).client_id + # credentials = var.credentials != null ? var.credentials : file(var.credentials_file) + # client_email = jsondecode(local.credentials).client_email + # client_id = jsondecode(local.credentials).client_id env = { defaults = { @@ -38,8 +38,8 @@ data "external" "git_checkout" { } provider "google" { - credentials = local.credentials - # credentials = file("~/.config/gcloud/application_default_credentials.json") + # credentials = local.credentials + credentials = file("~/.config/gcloud/application_default_credentials.json") project = var.project region = var.region @@ -59,8 +59,8 @@ resource "google_service_account_iam_binding" "serivce-account-iam" { role = "roles/iam.serviceAccountUser" members = [ - "serviceAccount:${local.client_email}", - # "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" + # "serviceAccount:${local.client_email}", + "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" ] } @@ -105,22 +105,22 @@ module "mpc-signer-lb" { count = length(var.signer_configs) source = "../modules/internal_cloudrun_lb" - name = "mpc-dev-signer-${count.index}" + name = "mpc-dev-signer-${count.index}-${var.env}" network_id = data.google_compute_network.prod_network.id subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id project_id = var.project region = "us-central1" - service_name = "mpc-recovery-signer-${count.index}-dev" + service_name = "mpc-recovery-signer-${count.index}-${var.env}" } module "mpc-leader-lb" { source = "../modules/internal_cloudrun_lb" - name = "mpc-dev-leader" + name = "mpc-dev-leader-${var.env}" network_id = data.google_compute_network.prod_network.id subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id project_id = var.project region = "us-central1" - service_name = "mpc-recovery-leader-dev" + service_name = "mpc-recovery-leader-${var.env}" } /* * Create multiple signer nodes @@ -129,8 +129,8 @@ module "signer" { count = length(var.signer_configs) source = "../modules/signer" - env = "dev" - service_name = "mpc-recovery-signer-${count.index}-dev" + env = var.env + service_name = "mpc-recovery-signer-${count.index}-${var.env}" project = var.project region = var.region zone = var.zone @@ -157,8 +157,8 @@ module "signer" { module "leader" { source = "../modules/leader" - env = "dev" - service_name = "mpc-recovery-leader-dev" + env = var.env + service_name = "mpc-recovery-leader-${var.env}" project = var.project region = var.region zone = var.zone diff --git a/infra/mpc-recovery-prod/main.tf b/infra/mpc-recovery-prod/main.tf index 948d5d5a1..de7fd286c 100644 --- a/infra/mpc-recovery-prod/main.tf +++ b/infra/mpc-recovery-prod/main.tf @@ -169,9 +169,8 @@ module "signer-mainnet" { node_id = count.index - oidc_providers_secret_id = var.oidc_providers_secret_id - cipher_key_secret_id = var.signer_configs[count.index].cipher_key_secret_id - sk_share_secret_id = var.signer_configs[count.index].sk_share_secret_id + cipher_key_secret_id = var.signer_configs[count.index].cipher_key_secret_id + sk_share_secret_id = var.signer_configs[count.index].sk_share_secret_id depends_on = [ google_secret_manager_secret_iam_member.cipher_key_secret_access, @@ -196,9 +195,8 @@ module "signer-testnet" { node_id = count.index - oidc_providers_secret_id = var.oidc_providers_secret_id - cipher_key_secret_id = var.signer_configs[count.index].cipher_key_secret_id - sk_share_secret_id = var.signer_configs[count.index].sk_share_secret_id + cipher_key_secret_id = var.signer_configs[count.index].cipher_key_secret_id + sk_share_secret_id = var.signer_configs[count.index].sk_share_secret_id depends_on = [ google_secret_manager_secret_iam_member.cipher_key_secret_access, diff --git a/infra/mpc-recovery-prod/terraform-dev.tfvars b/infra/mpc-recovery-prod/terraform-dev.tfvars index 8196fbe68..6cfb52674 100644 --- a/infra/mpc-recovery-prod/terraform-dev.tfvars +++ b/infra/mpc-recovery-prod/terraform-dev.tfvars @@ -1,22 +1,22 @@ -env = "dev" +env = "mainnet" project = "pagoda-discovery-platform-dev" -docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:f405e5594e666ef6a6e5ccf701c0d70c9673d4fe" +docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-prod/mpc-recovery-mainnet/mpc-recovery-mainnet@sha256:3d2c6d9ab13cfd6ef4b5ea8d6ff03f4373b6d3abd1d12ce91c88034eb5f40548" -account_creator_id = "mpc-recovery-dev-creator.testnet" -account_creator_sk_secret_id = "mpc-account-creator-sk-dev" -oidc_providers_secret_id = "mpc-allowed-oidc-providers-dev" -fast_auth_partners_secret_id = "mpc-fast-auth-partners-dev" +account_creator_id = "account_creator.near" +account_creator_sk_secret_id = "mpc-account-creator-sk-mainnet" +oidc_providers_secret_id = "mpc-allowed-oidc-providers-mainnet" +fast_auth_partners_secret_id = "mpc-fast-auth-partners-mainnet" signer_configs = [ { - cipher_key_secret_id = "mpc-cipher-0-dev" - sk_share_secret_id = "mpc-sk-share-0-dev" + cipher_key_secret_id = "mpc-cipher-0-mainnet" + sk_share_secret_id = "mpc-sk-share-0-mainnet" }, { - cipher_key_secret_id = "mpc-cipher-1-dev" - sk_share_secret_id = "mpc-sk-share-1-dev" + cipher_key_secret_id = "mpc-cipher-1-mainnet" + sk_share_secret_id = "mpc-sk-share-1-mainnet" }, { - cipher_key_secret_id = "mpc-cipher-2-dev" - sk_share_secret_id = "mpc-sk-share-2-dev" + cipher_key_secret_id = "mpc-cipher-2-mainnet" + sk_share_secret_id = "mpc-sk-share-2-mainnet" } ] From 111cbef3c9e84de0d070bf4d6ce3d75dbb7337ba Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Tue, 7 Nov 2023 09:13:32 -0700 Subject: [PATCH 13/20] reverted credential selection --- infra/mpc-recovery-dev/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/infra/mpc-recovery-dev/main.tf b/infra/mpc-recovery-dev/main.tf index ef160f5e1..02fbe2280 100644 --- a/infra/mpc-recovery-dev/main.tf +++ b/infra/mpc-recovery-dev/main.tf @@ -13,9 +13,9 @@ terraform { } locals { - # credentials = var.credentials != null ? var.credentials : file(var.credentials_file) - # client_email = jsondecode(local.credentials).client_email - # client_id = jsondecode(local.credentials).client_id + credentials = var.credentials != null ? var.credentials : file(var.credentials_file) + client_email = jsondecode(local.credentials).client_email + client_id = jsondecode(local.credentials).client_id env = { defaults = { @@ -38,8 +38,8 @@ data "external" "git_checkout" { } provider "google" { - # credentials = local.credentials - credentials = file("~/.config/gcloud/application_default_credentials.json") + credentials = local.credentials + # credentials = file("~/.config/gcloud/application_default_credentials.json") project = var.project region = var.region @@ -59,8 +59,8 @@ resource "google_service_account_iam_binding" "serivce-account-iam" { role = "roles/iam.serviceAccountUser" members = [ - # "serviceAccount:${local.client_email}", - "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" + "serviceAccount:${local.client_email}", + # "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" ] } From b604bee309228914327853f2c306bf7676c0a9bf Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Tue, 7 Nov 2023 09:17:30 -0700 Subject: [PATCH 14/20] made SA dynamic again --- infra/mpc-recovery-dev/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/mpc-recovery-dev/main.tf b/infra/mpc-recovery-dev/main.tf index 02fbe2280..091ad8372 100644 --- a/infra/mpc-recovery-dev/main.tf +++ b/infra/mpc-recovery-dev/main.tf @@ -50,8 +50,8 @@ provider "google" { * Create brand new service account with basic IAM */ resource "google_service_account" "service_account" { - account_id = "mpc-recovery-dev" - display_name = "MPC Recovery dev Account" + account_id = "mpc-recovery-${var.env}" + display_name = "MPC Recovery ${var.env} Account" } resource "google_service_account_iam_binding" "serivce-account-iam" { From 13572f0818424ccead8ec6d327cb5a89fb1489b0 Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Fri, 10 Nov 2023 12:00:22 -0700 Subject: [PATCH 15/20] reverted GHA workflows --- .github/workflows/terraform-dev.yml | 6 ++++-- .github/workflows/terraform-feature-env.yml | 14 +++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/terraform-dev.yml b/.github/workflows/terraform-dev.yml index c7e1a4cef..4bef552b9 100644 --- a/.github/workflows/terraform-dev.yml +++ b/.github/workflows/terraform-dev.yml @@ -64,7 +64,8 @@ jobs: id: plan run: | terraform plan -input=false -no-color -lock-timeout=1h -var-file terraform-dev.tfvars \ - -var "credentials=$GOOGLE_CREDENTIALS" + -var "credentials=$GOOGLE_CREDENTIALS" \ + -var docker_image=us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} @@ -140,6 +141,7 @@ jobs: if: github.ref == 'refs/heads/develop' && github.event_name == 'push' run: | terraform apply -auto-approve -input=false -lock-timeout=1h -var-file terraform-dev.tfvars \ - -var "credentials=$GOOGLE_CREDENTIALS" + -var "credentials=$GOOGLE_CREDENTIALS" \ + -var docker_image=us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} diff --git a/.github/workflows/terraform-feature-env.yml b/.github/workflows/terraform-feature-env.yml index 2e3d4a4dc..35c8c7efa 100644 --- a/.github/workflows/terraform-feature-env.yml +++ b/.github/workflows/terraform-feature-env.yml @@ -42,10 +42,13 @@ jobs: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} PR_NUMBER: ${{ env.PR_NUMBER }} - # Build Docker image. - - name: Docker Image - id: build - run: docker build .. -t near/mpc-recovery + - name: Wait for Docker Image to be Ready + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: 'Build and Push' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 10 # Applies Terraform configuration to the temporary environment - name: Terraform Apply @@ -53,7 +56,8 @@ jobs: run: | terraform apply -auto-approve -input=false -no-color -lock-timeout=1h -var-file terraform-dev.tfvars \ -var "credentials=$GOOGLE_CREDENTIALS" \ - -var "env=dev-$PR_NUMBER" + -var "env=dev-$PR_NUMBER" \ + -var docker_image=us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:${{ github.sha }} env: GOOGLE_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS_DEV }} PR_NUMBER: ${{ env.PR_NUMBER }} From 3beb83318ee253e086681549edfdc30938a56fc5 Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Fri, 10 Nov 2023 12:04:39 -0700 Subject: [PATCH 16/20] added back jwt signer url --- infra/modules/signer/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/modules/signer/main.tf b/infra/modules/signer/main.tf index b4f704d1c..7800b806d 100644 --- a/infra/modules/signer/main.tf +++ b/infra/modules/signer/main.tf @@ -56,6 +56,10 @@ resource "google_cloud_run_v2_service" "signer" { } } } + env { + name = "MPC_RECOVERY_JWT_SIGNATURE_PK_URL" + value = var.jwt_signature_pk_url + } env { name = "RUST_LOG" value = "mpc_recovery=debug" From 442da2d3b9391787a0981ceaf386d37f6a5d1014 Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Tue, 14 Nov 2023 12:23:24 -0800 Subject: [PATCH 17/20] updated prod env to include OTLP data --- infra/README.md | 20 ++++++++++++++++++ infra/mpc-recovery-prod/main.tf | 4 ++++ infra/mpc-recovery-prod/terraform-dev.tfvars | 22 -------------------- infra/mpc-recovery-prod/variables.tf | 8 +++++++ 4 files changed, 32 insertions(+), 22 deletions(-) create mode 100644 infra/README.md delete mode 100644 infra/mpc-recovery-prod/terraform-dev.tfvars diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 000000000..8051df6ed --- /dev/null +++ b/infra/README.md @@ -0,0 +1,20 @@ +# MPC Recovery Infrastructure Overview + +There are currently 3 mostly static environments for MPC + - Mainnet (production) + - Testnet (production) + - Dev (development) + + ## Mainnet/Testnet + + Mainnet and Testnet infra code is in the directory `mpc-recovery-prod` and is built off of the `main` GitHub Branch + - This environment should be deployed via the GHA pipeline `deploy-prod.yml` manually in order to prevent unwanted changes + - Both Mainnet and Testnet are treated as production environments + + ## Dev + + The Dev environment infra code is located in the `mpc-recovery-dev` directory and is built off of the `develop` GitHub Branch + - This should be used as the main development environment + - Every time a pull request is opened up against the `develop` branch, a new, ephemeral environment is created with your changes + - *Note: These environments will have the associated PR number appended to all resources* + - When a pull request is approved and merged into the `develop` branch, a new revision is deployed to the static Dev environment with the PRs changes and the PRs ephemeral environment is destroyed \ No newline at end of file diff --git a/infra/mpc-recovery-prod/main.tf b/infra/mpc-recovery-prod/main.tf index de7fd286c..a43b67d78 100644 --- a/infra/mpc-recovery-prod/main.tf +++ b/infra/mpc-recovery-prod/main.tf @@ -220,6 +220,8 @@ module "leader-mainnet" { docker_image = var.docker_image connector_id = var.prod-connector jwt_signature_pk_url = var.jwt_signature_pk_url + opentelemetry_level = var.opentelemetry_level + otlp_endpoint = var.otlp_endpoint signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) near_rpc = local.workspace.near_rpc @@ -248,6 +250,8 @@ module "leader-testnet" { docker_image = var.docker_image connector_id = var.prod-connector jwt_signature_pk_url = var.jwt_signature_pk_url + opentelemetry_level = var.opentelemetry_level + otlp_endpoint = var.otlp_endpoint signer_node_urls = concat(module.signer.*.node.uri, var.external_signer_node_urls) near_rpc = local.workspace.near_rpc diff --git a/infra/mpc-recovery-prod/terraform-dev.tfvars b/infra/mpc-recovery-prod/terraform-dev.tfvars deleted file mode 100644 index 6cfb52674..000000000 --- a/infra/mpc-recovery-prod/terraform-dev.tfvars +++ /dev/null @@ -1,22 +0,0 @@ -env = "mainnet" -project = "pagoda-discovery-platform-dev" -docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-prod/mpc-recovery-mainnet/mpc-recovery-mainnet@sha256:3d2c6d9ab13cfd6ef4b5ea8d6ff03f4373b6d3abd1d12ce91c88034eb5f40548" - -account_creator_id = "account_creator.near" -account_creator_sk_secret_id = "mpc-account-creator-sk-mainnet" -oidc_providers_secret_id = "mpc-allowed-oidc-providers-mainnet" -fast_auth_partners_secret_id = "mpc-fast-auth-partners-mainnet" -signer_configs = [ - { - cipher_key_secret_id = "mpc-cipher-0-mainnet" - sk_share_secret_id = "mpc-sk-share-0-mainnet" - }, - { - cipher_key_secret_id = "mpc-cipher-1-mainnet" - sk_share_secret_id = "mpc-sk-share-1-mainnet" - }, - { - cipher_key_secret_id = "mpc-cipher-2-mainnet" - sk_share_secret_id = "mpc-sk-share-2-mainnet" - } -] diff --git a/infra/mpc-recovery-prod/variables.tf b/infra/mpc-recovery-prod/variables.tf index 46ee89ab5..f48a2021f 100644 --- a/infra/mpc-recovery-prod/variables.tf +++ b/infra/mpc-recovery-prod/variables.tf @@ -86,4 +86,12 @@ data "google_compute_network" "prod_network" { variable "jwt_signature_pk_url" { type = string +} + +variable "otlp_endpoint" { + type = string +} + +variable "opentelemetry_level" { + type = string } \ No newline at end of file From cd8b8ddf9638dd19703922cf2664e3a3d64a2505 Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Tue, 14 Nov 2023 13:56:01 -0800 Subject: [PATCH 18/20] added correct naming convention for LB stuff --- infra/mpc-recovery-dev/main.tf | 18 +++++++++--------- infra/mpc-recovery-dev/terraform-dev.tfvars | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/infra/mpc-recovery-dev/main.tf b/infra/mpc-recovery-dev/main.tf index 091ad8372..842ee018d 100644 --- a/infra/mpc-recovery-dev/main.tf +++ b/infra/mpc-recovery-dev/main.tf @@ -13,9 +13,9 @@ terraform { } locals { - credentials = var.credentials != null ? var.credentials : file(var.credentials_file) - client_email = jsondecode(local.credentials).client_email - client_id = jsondecode(local.credentials).client_id + # credentials = var.credentials != null ? var.credentials : file(var.credentials_file) + # client_email = jsondecode(local.credentials).client_email + # client_id = jsondecode(local.credentials).client_id env = { defaults = { @@ -38,8 +38,8 @@ data "external" "git_checkout" { } provider "google" { - credentials = local.credentials - # credentials = file("~/.config/gcloud/application_default_credentials.json") + # credentials = local.credentials + credentials = file("~/.config/gcloud/application_default_credentials.json") project = var.project region = var.region @@ -59,8 +59,8 @@ resource "google_service_account_iam_binding" "serivce-account-iam" { role = "roles/iam.serviceAccountUser" members = [ - "serviceAccount:${local.client_email}", - # "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" + # "serviceAccount:${local.client_email}", + "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" ] } @@ -105,7 +105,7 @@ module "mpc-signer-lb" { count = length(var.signer_configs) source = "../modules/internal_cloudrun_lb" - name = "mpc-dev-signer-${count.index}-${var.env}" + name = "mpc-${var.env}-signer-${count.index}" network_id = data.google_compute_network.prod_network.id subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id project_id = var.project @@ -115,7 +115,7 @@ module "mpc-signer-lb" { module "mpc-leader-lb" { source = "../modules/internal_cloudrun_lb" - name = "mpc-dev-leader-${var.env}" + name = "mpc-${var.env}-leader" network_id = data.google_compute_network.prod_network.id subnetwork_id = data.google_compute_subnetwork.prod_subnetwork.id project_id = var.project diff --git a/infra/mpc-recovery-dev/terraform-dev.tfvars b/infra/mpc-recovery-dev/terraform-dev.tfvars index 8568423aa..fb2eaef81 100644 --- a/infra/mpc-recovery-dev/terraform-dev.tfvars +++ b/infra/mpc-recovery-dev/terraform-dev.tfvars @@ -3,7 +3,7 @@ project = "pagoda-discovery-platform-dev" docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:f405e5594e666ef6a6e5ccf701c0d70c9673d4fe" account_creator_id = "mpc-recovery-dev-creator.testnet" -account_creator_sk_secret_id = "mpc-recovery-account-creator-sk-dev" +account_creator_sk_secret_id = "mpc-account-creator-sk-dev" fast_auth_partners_secret_id = "mpc-fast-auth-partners-dev" signer_configs = [ { From 08d3439ec5587886b0d66d34212b6f84222c56ef Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Tue, 14 Nov 2023 13:57:44 -0800 Subject: [PATCH 19/20] added correct credential info --- infra/mpc-recovery-dev/main.tf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/infra/mpc-recovery-dev/main.tf b/infra/mpc-recovery-dev/main.tf index 842ee018d..c6865e82b 100644 --- a/infra/mpc-recovery-dev/main.tf +++ b/infra/mpc-recovery-dev/main.tf @@ -13,9 +13,9 @@ terraform { } locals { - # credentials = var.credentials != null ? var.credentials : file(var.credentials_file) - # client_email = jsondecode(local.credentials).client_email - # client_id = jsondecode(local.credentials).client_id + credentials = var.credentials != null ? var.credentials : file(var.credentials_file) + client_email = jsondecode(local.credentials).client_email + client_id = jsondecode(local.credentials).client_id env = { defaults = { @@ -38,8 +38,8 @@ data "external" "git_checkout" { } provider "google" { - # credentials = local.credentials - credentials = file("~/.config/gcloud/application_default_credentials.json") + credentials = local.credentials + # credentials = file("~/.config/gcloud/application_default_credentials.json") project = var.project region = var.region @@ -59,8 +59,8 @@ resource "google_service_account_iam_binding" "serivce-account-iam" { role = "roles/iam.serviceAccountUser" members = [ - # "serviceAccount:${local.client_email}", - "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" + "serviceAccount:${local.client_email}", + # "serviceAccount:mpc-recovery@pagoda-discovery-platform-dev.iam.gserviceaccount.com" ] } From b85d5c77ee811b41f52350529db5012d402d234e Mon Sep 17 00:00:00 2001 From: kmaus-near Date: Tue, 14 Nov 2023 14:16:15 -0800 Subject: [PATCH 20/20] added correct credential info --- infra/mpc-recovery-dev/terraform-dev.tfvars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/mpc-recovery-dev/terraform-dev.tfvars b/infra/mpc-recovery-dev/terraform-dev.tfvars index fb2eaef81..8568423aa 100644 --- a/infra/mpc-recovery-dev/terraform-dev.tfvars +++ b/infra/mpc-recovery-dev/terraform-dev.tfvars @@ -3,7 +3,7 @@ project = "pagoda-discovery-platform-dev" docker_image = "us-east1-docker.pkg.dev/pagoda-discovery-platform-dev/mpc-recovery/mpc-recovery-dev:f405e5594e666ef6a6e5ccf701c0d70c9673d4fe" account_creator_id = "mpc-recovery-dev-creator.testnet" -account_creator_sk_secret_id = "mpc-account-creator-sk-dev" +account_creator_sk_secret_id = "mpc-recovery-account-creator-sk-dev" fast_auth_partners_secret_id = "mpc-fast-auth-partners-dev" signer_configs = [ {