feat: Add optional external_bucket var for cross-cloud instance level buckets

[levinandrew]

Below the terraform plan you can see the updated env vars. Also note, tested logging a run file and artifact to this bucket to verify the queue functionality worked as expected still.

TF plan:

Terraform will perform the following actions:

  # module.wandb.data.google_compute_forwarding_rules.all will be read during apply
  # (depends on a resource or a module with changes pending)
 <= data "google_compute_forwarding_rules" "all" {
      + id    = (known after apply)
      + rules = (known after apply)
    }

  # module.wandb.module.gke_app.kubernetes_deployment.wandb will be updated in-place
  ~ resource "kubernetes_deployment" "wandb" {
        id               = "default/wandb"
        # (1 unchanged attribute hidden)

      ~ spec {
            # (5 unchanged attributes hidden)

          ~ template {
              ~ spec {
                    # (14 unchanged attributes hidden)

                  ~ container {
                        name                       = "wandb"
                        # (9 unchanged attributes hidden)

                      ~ env {
                            name  = "BUCKET"
                          ~ value = "gs://andrew-levin-gcp-humble-mule" -> "gs://"
                        }

                        # (29 unchanged blocks hidden)
                    }

         ...
    }

  # module.wandb.module.wandb.helm_release.wandb will be updated in-place
  ~ resource "helm_release" "wandb" {
        id                         = "wandb-cr"
      ~ metadata                   = [
          - {
...
              - values         = jsonencode(
                    {
                      - name = "wandb"
                      - spec = <<-EOT
                            "values":
                              "app":
                                "extraEnvs": {}
                                "serviceAccount":
                                  "annotations": {}
                                  "name": ""
                              "flat-runs-fields-updater":
                                "serviceAccount":
                                  "annotations": {}
                                  "name": null
                              "global":
                                "bucket":
                                  "name": "andrew-levin-gcp-humble-mule"
                                  "path": "nested/path"
                                  "provider": "gcs"
                                "cloudProvider": "gcp"
 ...
                        EOT
                    }
                )
              - version        = "0.1.0"
            },
        ] -> (known after apply)
        name                       = "wandb-cr"
        # (26 unchanged attributes hidden)

      - set {
          # At least one attribute in this block is (or was) sensitive,
          # so its contents will not be displayed.
        }
      + set {
          # At least one attribute in this block is (or was) sensitive,
          # so its contents will not be displayed.
        }

        # (1 unchanged block hidden)
    }

  # module.wandb.module.storage[0].module.bucket.google_storage_bucket.file_storage will be destroyed
  # (because module.wandb.module.storage[0].module.bucket is not in configuration)
  - resource "google_storage_bucket" "file_storage" {
...
      - url                         = "gs://andrew-levin-gcp-humble-mule" -> null
...
    }

  # module.wandb.module.storage[0].module.bucket.google_storage_bucket_iam_member.object_admin will be destroyed
  # (because module.wandb.module.storage[0].module.bucket is not in configuration)
  - resource "google_storage_bucket_iam_member" "object_admin" {
      - bucket = "b/andrew-levin-gcp-humble-mule" -> null
      - etag   = "CAI=" -> null
      - id     = "b/andrew-levin-gcp-humble-mule/roles/storage.objectAdmin/serviceAccount:<redactedServiceAccountEmail>" -> null
      - member = "serviceAccount:<redactedServiceAccountEmail>" -> null
      - role   = "roles/storage.objectAdmin" -> null
    }

  # module.wandb.module.storage[0].module.bucket.random_pet.file_storage will be destroyed
  # (because module.wandb.module.storage[0].module.bucket is not in configuration)
  - resource "random_pet" "file_storage" {
      - id        = "humble-mule" -> null
      - length    = 2 -> null
      - separator = "-" -> null
    }

Plan: 0 to add, 2 to change, 3 to destroy.

Changes to Outputs:
  ~ bucket_name            = "andrew-levin-gcp-humble-mule" -> "crosscloudsfs/ccsfstest"
  + bucket_queue_name      = "internal://"

Updated env vars:

wandb@wandb-app-84469f859f-zczl7:~$ env | grep -i bucket
BUCKET=az://crosscloudsfs/ccsfstest
BUCKET_QUEUE=internal://
  "overflow-bucket": {
OVERFLOW_BUCKET_ADDR=az://crosscloudsfs/ccsfstest
GORILLA_STORAGE_BUCKET=az://crosscloudsfs/ccsfstest