From 65ed7154f7c2ebee81e549c8bedfe4bfc57a44ae Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Wed, 18 Dec 2024 16:13:46 -0600 Subject: [PATCH] add changes for stackgres 1.14 Signed-off-by: Jesse Nelson --- charts/hedera-mirror-common/Chart.yaml | 2 +- charts/hedera-mirror-common/values.yaml | 45 ++++++- .../stackgres/stackgres-cluster.yaml | 15 ++- charts/hedera-mirror/values-prod.yaml | 4 +- charts/hedera-mirror/values.yaml | 10 +- .../scripts/restore-stackgres-backup.sh | 103 ++------------- .../scripts/restore-volume-snapshot.sh | 118 ++++++------------ docs/runbook/scripts/utils.sh | 106 ++++++++++++++-- 8 files changed, 210 insertions(+), 193 deletions(-) diff --git a/charts/hedera-mirror-common/Chart.yaml b/charts/hedera-mirror-common/Chart.yaml index d12e81f1b67..d9fa56dab17 100644 --- a/charts/hedera-mirror-common/Chart.yaml +++ b/charts/hedera-mirror-common/Chart.yaml @@ -27,7 +27,7 @@ dependencies: condition: stackgres.enabled name: stackgres-operator repository: https://stackgres.io/downloads/stackgres-k8s/stackgres/helm/ - version: 1.13.0 + version: 1.14.1 - condition: traefik.enabled name: traefik repository: https://helm.traefik.io/traefik diff --git a/charts/hedera-mirror-common/values.yaml b/charts/hedera-mirror-common/values.yaml index 16ce61b9657..41ca543a3fe 100644 --- a/charts/hedera-mirror-common/values.yaml +++ b/charts/hedera-mirror-common/values.yaml @@ -417,10 +417,53 @@ promtail: operator: Exists stackgres: + collector: + config: + exporters: + prometheus: + metric_expiration: 2m + service: + pipelines: + metrics: + receivers: + - prometheus/prometheus # This appears to be corrected to just prometheus in 1.15+ and can be updated (possibly removed) when upgrading + resources: + limits: + memory: "4Gi" + cpu: "1500m" + requests: + memory: "3Gi" + cpu: "1000m" + prometheusOperator: + monitors: + - name: mirror-prometheus-prometheus + metadata: + namespace: common + name: mirror-prometheus-prometheus + spec: + podMetricsEndpoints: + - honorLabels: true + honorTimestamps: true + interval: 1m + metricRelabelings: + - action: drop + regex: ^(pg_stat.*_user.*|pg_table.*|pg_total_relation_size_bytes|pg.*)$ + sourceLabels: + - __name__ + path: /metrics + port: prom-http + scheme: https + tlsConfig: + ca: + secret: + key: tls.crt + name: mirror-prometheus-prometheus + serverName: stackgres-collector + enabled: false operator: image: - name: gcr.io/mirrornode/stackgres-operator + name: docker.io/xinatswirlds/stackgres-operator prometheusRules: enabled: true DatabaseStorageFull: diff --git a/charts/hedera-mirror/templates/stackgres/stackgres-cluster.yaml b/charts/hedera-mirror/templates/stackgres/stackgres-cluster.yaml index c5009b7bedf..02b7516b53c 100644 --- a/charts/hedera-mirror/templates/stackgres/stackgres-cluster.yaml +++ b/charts/hedera-mirror/templates/stackgres/stackgres-cluster.yaml @@ -6,14 +6,17 @@ metadata: name: {{ include "hedera-mirror.stackgres" . }} namespace: {{ include "hedera-mirror.namespace" . }} spec: - {{- if .Values.stackgres.backup.enabled }} configurations: + {{- if .Values.stackgres.backup.enabled }} backups: {{- range $backupConfig := .Values.stackgres.backup.instances }} - sgObjectStorage: {{ include "hedera-mirror.stackgres" $ }}-object-storage {{$backupConfig | toYaml | nindent 6}} {{- end }} - {{- end }} + {{- end }} + observability: + disableMetrics: {{ not .Values.stackgres.observability.enableMetricsExporter }} + prometheusAutobind: {{ .Values.stackgres.observability.enableMetricsExporter }} coordinator: configurations: sgPoolingConfig: {{ include "hedera-mirror.stackgres" . }}-coordinator @@ -23,7 +26,6 @@ spec: scripts: - sgScript: {{ include "hedera-mirror.stackgres" . }}-coordinator pods: - disableMetricsExporter: {{ not .Values.stackgres.coordinator.enableMetricsExporter }} disablePostgresUtil: {{ not .Values.stackgres.coordinator.enablePostgresUtil }} persistentVolume: {{ .Values.stackgres.coordinator.persistentVolume | toYaml | nindent 8 }} resources: @@ -51,9 +53,11 @@ spec: ssl: enabled: false # Disable SSL to work around https://github.com/hashgraph/hedera-mirror-node/issues/9143 version: {{ .Values.stackgres.postgresVersion | quote }} - prometheusAutobind: {{ or .Values.stackgres.coordinator.enableMetricsExporter .Values.stackgres.worker.enableMetricsExporter }} + prometheusAutobind: {{ .Values.stackgres.observability.enableMetricsExporter }} replication: - mode: {{ .Values.stackgres.coordinator.replication.mode }} + mode: {{ .Values.stackgres.replication.mode }} + initialization: + mode: {{ .Values.stackgres.replication.initialization.mode }} shards: clusters: {{ .Values.stackgres.worker.instances }} configurations: @@ -65,7 +69,6 @@ spec: - sgScript: {{ include "hedera-mirror.stackgres" . }}-worker overrides: {{ .Values.stackgres.worker.overrides | toYaml | nindent 6 }} pods: - disableMetricsExporter: {{ not .Values.stackgres.worker.enableMetricsExporter }} disablePostgresUtil: {{ not .Values.stackgres.worker.enablePostgresUtil }} persistentVolume: {{ .Values.stackgres.worker.persistentVolume | toYaml | nindent 8 }} resources: diff --git a/charts/hedera-mirror/values-prod.yaml b/charts/hedera-mirror/values-prod.yaml index e27f6c5dc42..dd374f5d346 100644 --- a/charts/hedera-mirror/values-prod.yaml +++ b/charts/hedera-mirror/values-prod.yaml @@ -145,10 +145,10 @@ stackgres: cronSchedule: "45 22 * * *" compression: lz4 useVolumeSnapshot: true - fastVolumeSnapshot: false # Can be true once StackGres issue 2887 is resolved + fastVolumeSnapshot: true volumeSnapshotClass: zfs performance: - uploadDiskConcurrency: 1 # Can be 4 once StackGres issue 2887 is resolved + uploadDiskConcurrency: 1 objectStorage: endpoint: http://mirror-minio.common:9000 bucket: stackgres-backup diff --git a/charts/hedera-mirror/values.yaml b/charts/hedera-mirror/values.yaml index 35e22cb6334..da61d4ad588 100644 --- a/charts/hedera-mirror/values.yaml +++ b/charts/hedera-mirror/values.yaml @@ -310,7 +310,6 @@ stackgres: wal_init_zero: "off" # Not needed with ZFS wal_recycle: "off" # Not needed with ZFS work_mem: "12MB" - enableMetricsExporter: false enablePostgresUtil: true instances: 1 pgbouncer: @@ -325,8 +324,6 @@ stackgres: pool_mode: session mirror_importer: pool_mode: session - replication: - mode: sync-all resources: cpu: 100m memory: 1Gi @@ -342,9 +339,15 @@ stackgres: - name: pg_trgm version: "1.6" nameOverride: citus + observability: + enableMetricsExporter: true podAntiAffinity: true postgresVersion: "16.2" priorityClassName: "" + replication: + initialization: + mode: FromReplica + mode: sync-all worker: config: autovacuum_max_workers: "2" @@ -362,7 +365,6 @@ stackgres: wal_init_zero: "off" # Not needed with ZFS wal_recycle: "off" # Not needed with ZFS work_mem: "12MB" - enableMetricsExporter: false enablePostgresUtil: true instances: 1 overrides: [] # Override shard(s) configuration diff --git a/docs/runbook/scripts/restore-stackgres-backup.sh b/docs/runbook/scripts/restore-stackgres-backup.sh index 3a954c6fe38..5c0a089abe8 100755 --- a/docs/runbook/scripts/restore-stackgres-backup.sh +++ b/docs/runbook/scripts/restore-stackgres-backup.sh @@ -108,7 +108,8 @@ function createSGShardedClusterConfigWithRestore() { "downloadDiskConcurrency": 1 } } -EOF) +EOF +) CLUSTER_CONFIG=$(echo "${sourceConfig}" | \ jq --argjson initialDataConfig "${initialDataConfig}" '.spec.initialData=$initialDataConfig') log "Created SGShardedCluster configuration to restore backup ${BACKUP_TO_RESTORE}" @@ -144,75 +145,6 @@ function findShardedCluster() { doContinue } -function fixClusterAuth() { - local sgPasswords=$(kubectl get secret "${CLUSTER}" -o json | - ksd | - jq -r '.stringData') - local superuserUsername=$(echo "${sgPasswords}" | jq -r '.["superuser-username"]') - local superuserPassword=$(echo "${sgPasswords}" | jq -r '.["superuser-password"]') - local replicationUsername=$(echo "${sgPasswords}" | jq -r '.["replication-username"]') - local replicationPassword=$(echo "${sgPasswords}" | jq -r '.["replication-password"]') - local authenticatorUsername=$(echo "${sgPasswords}" | jq -r '.["authenticator-username"]') - local authenticatorPassword=$(echo "${sgPasswords}" | jq -r '.["authenticator-password"]') - - # Mirror Node Passwords - local mirrorNodePasswords=$(kubectl get secret "${HELM_RELEASE_NAME}-passwords" -o json | - ksd | - jq -r '.stringData') - local graphqlUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRAPHQL_DB_USERNAME') - local graphqlPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRAPHQL_DB_PASSWORD') - local grpcUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRPC_DB_USERNAME') - local grpcPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRPC_DB_PASSWORD') - local importerUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_USERNAME') - local importerPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_PASSWORD') - local ownerUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_OWNER') - local ownerPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD') - local restUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_REST_DB_USERNAME') - local restPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_REST_DB_PASSWORD') - local restJavaUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_RESTJAVA_DB_USERNAME') - local restJavaPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_RESTJAVA_DB_PASSWORD') - local rosettaUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_ROSETTA_DB_USERNAME') - local rosettaPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_ROSETTA_DB_PASSWORD') - local web3Username=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_WEB3_DB_USERNAME') - local web3Password=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_WEB3_DB_PASSWORD') - local dbName=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_NAME') - local sql=$(cat </dev/null 2>&1; do - sleep 1 - done - - expectedTotal=$(($(kubectl get sgshardedclusters "${CLUSTER}" -o jsonpath='{.spec.shards.clusters}')+1)) - while [[ "$(kubectl get sts -l 'app=StackGresCluster' -o name | wc -l)" -ne "${expectedTotal}" ]]; do - sleep 1 - done - - log "Waiting for all StackGresCluster pods to be ready" - for sts in $(kubectl get sts -l 'app=StackGresCluster' -o name); do - expected=$(kubectl get "${sts}" -o jsonpath='{.spec.replicas}') - kubectl wait --for=jsonpath='{.status.readyReplicas}'=${expected} "${sts}" --timeout=-1s - done -} - CURRENT_NAMESPACE=$(kubectl config view --minify --output 'jsonpath={..namespace}') prepare diff --git a/docs/runbook/scripts/restore-volume-snapshot.sh b/docs/runbook/scripts/restore-volume-snapshot.sh index 76519ba3495..fe0c7f6f5bd 100755 --- a/docs/runbook/scripts/restore-volume-snapshot.sh +++ b/docs/runbook/scripts/restore-volume-snapshot.sh @@ -8,7 +8,6 @@ REPLACE_DISKS="${REPLACE_DISKS:-true}" ZFS_POOL_NAME="${ZFS_POOL_NAME:-zfspv-pool}" function configureAndValidate() { - CURRENT_CONTEXT=$(kubectl config current-context) GCP_PROJECT="$(readUserInput "Enter GCP Project for target: ")" if [[ -z "${GCP_PROJECT}" ]]; then log "GCP_PROJECT is not set and is required. Exiting" @@ -120,6 +119,7 @@ function configureAndValidate() { function prepareDiskReplacement() { for namespace in "${NAMESPACES[@]}"; do unrouteTraffic "${namespace}" + kubectl delete sgshardedbackups.stackgres.io -n "${namespace}" --all pauseCitus "${namespace}" done @@ -159,7 +159,16 @@ function renameZfsVolumes() { log "Snapshot pvc ${ZFS_POOL_NAME}/${snapshotPvcVolumeName} already matches pvc ${ZFS_POOL_NAME}/${pvcVolumeName}" fi done - kubectl_common exec "${podInfo}" -c openebs-zfs-plugin -- zfs list + + local zfsSnapshots="$(kubectl_common exec "${podInfo}" -c openebs-zfs-plugin -- bash -c 'zfs list -H -o name -t snapshot')" + + if [[ -z "${zfsSnapshots}" ]]; then + log "No snapshots found for pool ${ZFS_POOL_NAME} and node ${nodeId}" + else + log "Deleting snapshots ${zfsSnapshots}" + kubectl_common exec "${podInfo}" -c openebs-zfs-plugin -- bash -c "echo \"${zfsSnapshots}\" | xargs -n1 zfs destroy" + fi + kubectl_common exec "${podInfo}" -c openebs-zfs-plugin -- zfs list -t filesystem,snapshot done log "ZFS datasets renamed" } @@ -201,23 +210,39 @@ function configureShardedClusterResource() { sort_by(.citusCluster.citusGroup, .citusCluster.podName)| to_entries| map({index: .key, pods: {persistentVolume: {size: .value.snapshotPvcSize}}})') - local shardedClusterPatch=$(echo "${workerPvcOverrides}" | - jq -r --arg coordinatorPvcSize "${coordinatorPvcSize}" \ - '{ + + log "Patching sharded cluster ${shardedClusterName} in namespace ${namespace}" + local shardedCluster=$(kubectl get sgshardedclusters.stackgres.io -n "${namespace}" "${shardedClusterName}" -o json) + + log "Deleting backup configuration if present"; + echo "${shardedCluster}" | jq -r 'del(.spec.configurations.backups)' | kubectl apply -f - + local shardedClusterPatch=$(echo "${shardedCluster} ${workerPvcOverrides}" | + jq -s --arg COORDINATOR_PVC_SIZE "${coordinatorPvcSize}" \ + '.[0] as $cluster | + .[1] as $overrides | + $cluster | + if(.spec.configurations | has("backups")) + then .spec.configurations.backups | map(del(.paths)) + else + [] + end | + { spec: { + configurations: { + backups: (.) + }, coordinator: { pods: { persistentVolume: { - size: $coordinatorPvcSize + size: $COORDINATOR_PVC_SIZE } } }, shards: { - overrides: (.) + overrides: $overrides } - } - }') - log "Patching sharded cluster ${shardedClusterName} in namespace ${namespace} with ${shardedClusterPatch}" + } + }') kubectl patch sgshardedclusters.stackgres.io -n "${namespace}" "${shardedClusterName}" --type merge -p "${shardedClusterPatch}" log " **** IMPORTANT **** @@ -234,55 +259,11 @@ function configureShardedClusterResource() { function markAndConfigurePrimaries() { local pvcsInNamespace="${1}" local shardedClusterName="${2}" + local namespace="${3}" - # Stackgres Passwords local primaryCoordinator=$(echo "${pvcsInNamespace}" | jq -r 'map(select(.snapshotPrimary and .citusCluster.isCoordinator))|first') - local sgPasswordsSecretName=$(echo "${primaryCoordinator}" | jq -r '.citusCluster.clusterName') - local sgPasswords=$(kubectl get secret -n "${namespace}" "${sgPasswordsSecretName}" -o json | - ksd | - jq -r '.stringData') - local superuserUsername=$(echo "${sgPasswords}" | jq -r '.["superuser-username"]') - local superuserPassword=$(echo "${sgPasswords}" | jq -r '.["superuser-password"]') - local replicationUsername=$(echo "${sgPasswords}" | jq -r '.["replication-username"]') - local replicationPassword=$(echo "${sgPasswords}" | jq -r '.["replication-password"]') - local authenticatorUsername=$(echo "${sgPasswords}" | jq -r '.["authenticator-username"]') - local authenticatorPassword=$(echo "${sgPasswords}" | jq -r '.["authenticator-password"]') - - # Mirror Node Passwords - local mirrorNodePasswords=$(kubectl get secret -n "${namespace}" "${HELM_RELEASE_NAME}-passwords" -o json | - ksd | - jq -r '.stringData') - local graphqlUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRAPHQL_DB_USERNAME') - local graphqlPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRAPHQL_DB_PASSWORD') - local grpcUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRPC_DB_USERNAME') - local grpcPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRPC_DB_PASSWORD') - local importerUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_USERNAME') - local importerPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_PASSWORD') - local ownerUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_OWNER') - local ownerPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD') - local restUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_REST_DB_USERNAME') - local restPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_REST_DB_PASSWORD') - local restJavaUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_RESTJAVA_DB_USERNAME') - local restJavaPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_RESTJAVA_DB_PASSWORD') - local rosettaUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_ROSETTA_DB_USERNAME') - local rosettaPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_ROSETTA_DB_PASSWORD') - local web3Username=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_WEB3_DB_USERNAME') - local web3Password=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_WEB3_DB_PASSWORD') - local dbName=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_NAME') - local updatePasswordsSql=$(cat </dev/null 2>&1; do + sleep 1 + done + + expectedTotal=$(($(kubectl get sgshardedclusters -n "${namespace}" -o jsonpath='{.items[0].spec.shards.clusters}')+1)) + while [[ "$(kubectl get sts -n "${namespace}" -l 'app=StackGresCluster' -o name | wc -l)" -ne "${expectedTotal}" ]]; do + sleep 1 + done + + log "Waiting for all StackGresCluster pods to be ready" + for sts in $(kubectl get sts -n "${namespace}" -l 'app=StackGresCluster' -o name); do + expected=$(kubectl get "${sts}" -n "${namespace}" -o jsonpath='{.spec.replicas}') + kubectl wait --for=jsonpath='{.status.readyReplicas}'=${expected} "${sts}" -n "${namespace}" --timeout=-1s + done + + if [[ "${waitForMaster}" == "true" ]]; then + while [[ "$(kubectl get pods -n "${namespace}" -l "${STACKGRES_MASTER_LABELS}" -o name | wc -l)" -ne "${expectedTotal}" ]]; do + log "Waiting for all pods to be marked with master role label" + sleep 1 + done + fi else log "Citus is already running in namespace ${namespace}. Skipping" fi @@ -239,6 +258,76 @@ function resizeCitusNodePools() { fi } +function updateStackgresCreds() { + local cluster="${1}" + local namespace="${2}" + local sgPasswords=$(kubectl get secret -n "${namespace}" "${cluster}" -o json | + ksd | + jq -r '.stringData') + local superuserUsername=$(echo "${sgPasswords}" | jq -r '.["superuser-username"]') + local superuserPassword=$(echo "${sgPasswords}" | jq -r '.["superuser-password"]') + local replicationUsername=$(echo "${sgPasswords}" | jq -r '.["replication-username"]') + local replicationPassword=$(echo "${sgPasswords}" | jq -r '.["replication-password"]') + local authenticatorUsername=$(echo "${sgPasswords}" | jq -r '.["authenticator-username"]') + local authenticatorPassword=$(echo "${sgPasswords}" | jq -r '.["authenticator-password"]') + + # Mirror Node Passwords + local mirrorNodePasswords=$(kubectl get secret -n "${namespace}" "${HELM_RELEASE_NAME}-passwords" -o json | + ksd | + jq -r '.stringData') + local graphqlUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRAPHQL_DB_USERNAME') + local graphqlPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRAPHQL_DB_PASSWORD') + local grpcUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRPC_DB_USERNAME') + local grpcPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_GRPC_DB_PASSWORD') + local importerUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_USERNAME') + local importerPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_PASSWORD') + local ownerUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_OWNER') + local ownerPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD') + local restUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_REST_DB_USERNAME') + local restPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_REST_DB_PASSWORD') + local restJavaUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_RESTJAVA_DB_USERNAME') + local restJavaPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_RESTJAVA_DB_PASSWORD') + local rosettaUsername=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_ROSETTA_DB_USERNAME') + local rosettaPassword=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_ROSETTA_DB_PASSWORD') + local web3Username=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_WEB3_DB_USERNAME') + local web3Password=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_WEB3_DB_PASSWORD') + local dbName=$(echo "${mirrorNodePasswords}" | jq -r '.HEDERA_MIRROR_IMPORTER_DB_NAME') + local sql=$(cat <