From 9cd2c1b7b6a041fb58b4e9ca1dc4224a49a41dea Mon Sep 17 00:00:00 2001 From: Brandon Martin Date: Thu, 5 Dec 2024 15:11:31 -0800 Subject: [PATCH] Accept jdbcUrl as an environment variable (#32) * Accept jdbcUrl as an environment variable * CLI updates * Update metadata files --- .../app/services/AgroalDataSourceService.kt | 8 ++- .../kotlin/io/hasura/cli/IConfigGenerator.kt | 3 +- .../io/hasura/cli/MySQLConfigGenerator.kt | 10 ++- .../io/hasura/cli/OracleConfigGenerator.kt | 10 ++- .../io/hasura/cli/SnowflakeConfigGenerator.kt | 11 +++- ndc-cli/src/main/kotlin/io/hasura/cli/main.kt | 64 +++++++++++++------ .../.hasura-connector/connector-metadata.yaml | 3 +- .../.hasura-connector/connector-metadata.yaml | 3 +- .../.hasura-connector/connector-metadata.yaml | 3 +- .../ndc/common/ConnectorConfiguration.kt | 27 +++++++- 10 files changed, 110 insertions(+), 32 deletions(-) diff --git a/ndc-app/src/main/kotlin/io/hasura/ndc/app/services/AgroalDataSourceService.kt b/ndc-app/src/main/kotlin/io/hasura/ndc/app/services/AgroalDataSourceService.kt index 24a1f62..a2b796c 100644 --- a/ndc-app/src/main/kotlin/io/hasura/ndc/app/services/AgroalDataSourceService.kt +++ b/ndc-app/src/main/kotlin/io/hasura/ndc/app/services/AgroalDataSourceService.kt @@ -7,6 +7,7 @@ import io.agroal.api.configuration.supplier.AgroalDataSourceConfigurationSupplie import io.agroal.api.security.NamePrincipal import io.agroal.api.security.SimplePassword import io.hasura.ndc.common.ConnectorConfiguration +import io.hasura.ndc.common.JdbcUrlConfig import io.opentelemetry.instrumentation.annotations.WithSpan import io.opentelemetry.instrumentation.jdbc.datasource.OpenTelemetryDataSource import io.quarkus.agroal.runtime.AgroalOpenTelemetryWrapper @@ -174,7 +175,7 @@ class AgroalDataSourceService { } private fun mkAgroalDataSourceConfigurationSupplier( - jdbcUrl: String, + jdbcUrl: JdbcUrlConfig, properties: Map ) = AgroalDataSourceConfigurationSupplier().metricsEnabled().connectionPoolConfiguration { connectionPool -> @@ -193,7 +194,10 @@ class AgroalDataSourceService { .multipleAcquisition(config.connectionPoolConfiguration().multipleAcquisition()) .enhancedLeakReport(config.connectionPoolConfiguration().enhancedLeakReport()) .connectionFactoryConfiguration { connFactory -> - connFactory.jdbcUrl(jdbcUrl) + connFactory.jdbcUrl(when (jdbcUrl) { + is JdbcUrlConfig.Literal -> jdbcUrl.value + is JdbcUrlConfig.EnvVar -> System.getenv(jdbcUrl.variable) ?: throw IllegalArgumentException("Environment variable ${jdbcUrl.variable} not found") + }) connFactory.loginTimeout(config.connectionFactoryConfiguration().loginTimeout()) // To explain what is going on here: // diff --git a/ndc-cli/src/main/kotlin/io/hasura/cli/IConfigGenerator.kt b/ndc-cli/src/main/kotlin/io/hasura/cli/IConfigGenerator.kt index 45e0829..ae46d3f 100644 --- a/ndc-cli/src/main/kotlin/io/hasura/cli/IConfigGenerator.kt +++ b/ndc-cli/src/main/kotlin/io/hasura/cli/IConfigGenerator.kt @@ -1,7 +1,8 @@ package io.hasura.cli import io.hasura.ndc.common.ConnectorConfiguration +import io.hasura.ndc.common.JdbcUrlConfig interface IConfigGenerator { - fun getConfig(jdbcUrl: String, schemas: List): ConnectorConfiguration + fun getConfig(jdbcUrlConfig: JdbcUrlConfig, schemas: List): ConnectorConfiguration } \ No newline at end of file diff --git a/ndc-cli/src/main/kotlin/io/hasura/cli/MySQLConfigGenerator.kt b/ndc-cli/src/main/kotlin/io/hasura/cli/MySQLConfigGenerator.kt index 2dfbe84..bced97e 100644 --- a/ndc-cli/src/main/kotlin/io/hasura/cli/MySQLConfigGenerator.kt +++ b/ndc-cli/src/main/kotlin/io/hasura/cli/MySQLConfigGenerator.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import io.hasura.ndc.common.ColumnSchemaRow import io.hasura.ndc.common.ConnectorConfiguration +import io.hasura.ndc.common.JdbcUrlConfig import io.hasura.ndc.common.TableSchemaRow import io.hasura.ndc.common.TableType import org.jooq.impl.DSL @@ -12,10 +13,15 @@ object MySQLConfigGenerator : IConfigGenerator { private val mapper = jacksonObjectMapper() override fun getConfig( - jdbcUrl: String, + jdbcUrl: JdbcUrlConfig, schemas: List ): ConnectorConfiguration { - val ctx = DSL.using(jdbcUrl) + val jdbcUrlString = when (jdbcUrl) { + is JdbcUrlConfig.Literal -> jdbcUrl.value + is JdbcUrlConfig.EnvVar -> System.getenv(jdbcUrl.variable) + ?: throw IllegalArgumentException("Environment variable ${jdbcUrl.variable} not found") + } + val ctx = DSL.using(jdbcUrlString) //language=MySQL val sql = """ diff --git a/ndc-cli/src/main/kotlin/io/hasura/cli/OracleConfigGenerator.kt b/ndc-cli/src/main/kotlin/io/hasura/cli/OracleConfigGenerator.kt index d0f709e..a4736a7 100644 --- a/ndc-cli/src/main/kotlin/io/hasura/cli/OracleConfigGenerator.kt +++ b/ndc-cli/src/main/kotlin/io/hasura/cli/OracleConfigGenerator.kt @@ -3,6 +3,7 @@ package io.hasura.cli import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import io.hasura.ndc.common.ConnectorConfiguration +import io.hasura.ndc.common.JdbcUrlConfig import io.hasura.ndc.common.TableSchemaRow import io.hasura.ndc.common.TableType import org.jooq.impl.DSL @@ -12,10 +13,15 @@ object OracleConfigGenerator : IConfigGenerator { private val mapper = jacksonObjectMapper() override fun getConfig( - jdbcUrl: String, + jdbcUrl: JdbcUrlConfig, schemas: List ): ConnectorConfiguration { - val ctx = DSL.using(jdbcUrl) + val jdbcUrlString = when (jdbcUrl) { + is JdbcUrlConfig.Literal -> jdbcUrl.value + is JdbcUrlConfig.EnvVar -> System.getenv(jdbcUrl.variable) + ?: throw IllegalArgumentException("Environment variable ${jdbcUrl.variable} not found") + } + val ctx = DSL.using(jdbcUrlString) //language=Oracle val baseTableSql = """ diff --git a/ndc-cli/src/main/kotlin/io/hasura/cli/SnowflakeConfigGenerator.kt b/ndc-cli/src/main/kotlin/io/hasura/cli/SnowflakeConfigGenerator.kt index adfc003..bf9a422 100644 --- a/ndc-cli/src/main/kotlin/io/hasura/cli/SnowflakeConfigGenerator.kt +++ b/ndc-cli/src/main/kotlin/io/hasura/cli/SnowflakeConfigGenerator.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import io.hasura.ndc.common.ColumnSchemaRow import io.hasura.ndc.common.ConnectorConfiguration +import io.hasura.ndc.common.JdbcUrlConfig import io.hasura.ndc.common.TableSchemaRow import io.hasura.ndc.common.TableType import io.hasura.ndc.ir.ForeignKeyConstraint @@ -13,11 +14,17 @@ object SnowflakeConfigGenerator : IConfigGenerator { private val mapper = jacksonObjectMapper() override fun getConfig( - jdbcUrl: String, + jdbcUrl: JdbcUrlConfig, schemas: List ): ConnectorConfiguration { + val jdbcUrlString = when (jdbcUrl) { + is JdbcUrlConfig.Literal -> jdbcUrl.value + is JdbcUrlConfig.EnvVar -> System.getenv(jdbcUrl.variable) + ?: throw IllegalArgumentException("Environment variable ${jdbcUrl.variable} not found") + } + // Don't use Arrow memory format so we don't need to --add-opens=java.base/java.nio=ALL-UNNAMED to the JVM - val modifiedJdbcUrl = jdbcUrl.find { it == '?' } + val modifiedJdbcUrl = jdbcUrlString.find { it == '?' } ?.let { "$jdbcUrl&JDBC_QUERY_RESULT_FORMAT=JSON" } ?: "$jdbcUrl?JDBC_QUERY_RESULT_FORMAT=JSON" diff --git a/ndc-cli/src/main/kotlin/io/hasura/cli/main.kt b/ndc-cli/src/main/kotlin/io/hasura/cli/main.kt index 9b3392c..70e25f1 100644 --- a/ndc-cli/src/main/kotlin/io/hasura/cli/main.kt +++ b/ndc-cli/src/main/kotlin/io/hasura/cli/main.kt @@ -1,7 +1,9 @@ package io.hasura.cli import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue import io.hasura.ndc.common.ConnectorConfiguration +import io.hasura.ndc.common.JdbcUrlConfig import picocli.CommandLine import picocli.CommandLine.* import java.io.File @@ -34,11 +36,11 @@ class CLI { ) fun update( @Parameters( - arity = "1", + arity = "0..1", paramLabel = "", - description = ["JDBC URL to connect to the Oracle database"] + description = ["JDBC URL to connect to the database (optional)"] ) - jdbcUrl: String, + jdbcUrlParam: String?, @Option( names = ["-o", "--outfile"], defaultValue = "configuration.json", @@ -60,14 +62,37 @@ class CLI { ) { val file = File(outfile) - println("Checking for configuration file at ${file.absolutePath}") - val existingConfig = file.let { - if (it.exists()) { - println("Existing configuration file detected") - mapper.readValue(it, ConnectorConfiguration::class.java) - } else { - println("Non-existent or empty configuration file detected") - ConnectorConfiguration() + // Parse existing configuration + val existingConfig = if (file.exists()) { + println("Existing configuration file detected at ${file.absolutePath}") + try { + mapper.readValue(file) + } catch (e: Exception) { + println("Error reading existing configuration: ${e.message}") + null + } + } else { + println("No existing configuration file found at ${file.absolutePath}") + null + } + + // Determine the JDBC URL configuration + val jdbcUrlConfig = when { + jdbcUrlParam != null -> { + // If jdbcUrlParam is provided, use it as a literal value + JdbcUrlConfig.Literal(jdbcUrlParam) + } + System.getenv("JDBC_URL") != null -> { + // If JDBC_URL environment variable is set, use it + JdbcUrlConfig.EnvVar("JDBC_URL") + } + existingConfig?.jdbcUrl != null -> { + // If there's an existing config, use its jdbcUrl (which could be either Literal or EnvVar) + existingConfig.jdbcUrl + } + else -> { + // If none of the above conditions are met, throw an error + throw IllegalArgumentException("No JDBC URL provided and no existing configuration found") } } @@ -77,18 +102,21 @@ class CLI { DatabaseType.SNOWFLAKE -> SnowflakeConfigGenerator } - println("Generating configuration for $database database...") + println("Introspecting database...") val introspectedConfig = configGenerator.getConfig( - jdbcUrl = jdbcUrl, - schemas = schemas ?: emptyList() + jdbcUrlConfig = jdbcUrlConfig, + schemas = schemas ?: existingConfig?.schemas ?: emptyList() ) - val mergedConfigWithNativeQueries = introspectedConfig.copy( - nativeQueries = existingConfig.nativeQueries + + val finalConfig = introspectedConfig.copy( + jdbcUrl = jdbcUrlConfig, + nativeQueries = existingConfig?.nativeQueries ?: emptyMap() ) try { - println("Writing configuration to ${file.absolutePath}") - mapper.writerWithDefaultPrettyPrinter().writeValue(file, mergedConfigWithNativeQueries) + println("Writing updated configuration to ${file.absolutePath}") + mapper.writerWithDefaultPrettyPrinter().writeValue(file, finalConfig) + println("Configuration updated successfully") } catch (e: Exception) { println("Error writing configuration to file: ${e.message}") diff --git a/ndc-connector-mysql/.hasura-connector/connector-metadata.yaml b/ndc-connector-mysql/.hasura-connector/connector-metadata.yaml index ca8e878..5db5db3 100644 --- a/ndc-connector-mysql/.hasura-connector/connector-metadata.yaml +++ b/ndc-connector-mysql/.hasura-connector/connector-metadata.yaml @@ -7,9 +7,10 @@ supportedEnvironmentVariables: commands: update: | docker run \ + -e JDBC_URL \ -e HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH \ -v ${HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH}:/app/output \ - ghcr.io/hasura/ndc-jvm-cli:v0.1.3 update $JDBC_URL \ + ghcr.io/hasura/ndc-jvm-cli:v0.1.3 update \ --database MYSQL \ --schemas $JDBC_SCHEMAS \ --outfile /app/output/configuration.json diff --git a/ndc-connector-oracle/.hasura-connector/connector-metadata.yaml b/ndc-connector-oracle/.hasura-connector/connector-metadata.yaml index 5c5a5df..8b4d156 100644 --- a/ndc-connector-oracle/.hasura-connector/connector-metadata.yaml +++ b/ndc-connector-oracle/.hasura-connector/connector-metadata.yaml @@ -9,9 +9,10 @@ supportedEnvironmentVariables: commands: update: | docker run \ + -e JDBC_URL \ -e HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH \ -v ${HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH}:/app/output \ - ghcr.io/hasura/ndc-jvm-cli:v0.1.3 update $JDBC_URL \ + ghcr.io/hasura/ndc-jvm-cli:v0.1.3 update \ --database ORACLE \ --schemas $JDBC_SCHEMAS \ --outfile /app/output/configuration.json diff --git a/ndc-connector-snowflake/.hasura-connector/connector-metadata.yaml b/ndc-connector-snowflake/.hasura-connector/connector-metadata.yaml index a8b09d5..a66e247 100644 --- a/ndc-connector-snowflake/.hasura-connector/connector-metadata.yaml +++ b/ndc-connector-snowflake/.hasura-connector/connector-metadata.yaml @@ -9,9 +9,10 @@ supportedEnvironmentVariables: commands: update: | docker run \ + -e JDBC_URL \ -e HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH \ -v ${HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH}:/app/output \ - ghcr.io/hasura/ndc-jvm-cli:v0.1.0 update $JDBC_URL \ + ghcr.io/hasura/ndc-jvm-cli:v0.1.0 update \ --database SNOWFLAKE \ --schemas $JDBC_SCHEMAS \ --outfile /app/output/configuration.json diff --git a/ndc-ir/src/main/kotlin/io/hasura/ndc/common/ConnectorConfiguration.kt b/ndc-ir/src/main/kotlin/io/hasura/ndc/common/ConnectorConfiguration.kt index 4e424a1..b40ae90 100644 --- a/ndc-ir/src/main/kotlin/io/hasura/ndc/common/ConnectorConfiguration.kt +++ b/ndc-ir/src/main/kotlin/io/hasura/ndc/common/ConnectorConfiguration.kt @@ -2,11 +2,34 @@ package io.hasura.ndc.common import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.annotation.JsonValue import java.io.File import java.nio.file.Path +@JsonDeserialize(using = JdbcUrlConfigDeserializer::class) +sealed class JdbcUrlConfig { + data class Literal(@JsonValue val value: String) : JdbcUrlConfig() + data class EnvVar(val variable: String) : JdbcUrlConfig() +} + +class JdbcUrlConfigDeserializer : JsonDeserializer() { + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): JdbcUrlConfig { + val node: JsonNode = p.codec.readTree(p) + return when { + node.isTextual -> JdbcUrlConfig.Literal(node.asText()) + node.isObject && node.has("variable") -> JdbcUrlConfig.EnvVar(node.get("variable").asText()) + else -> throw IllegalArgumentException("Invalid JdbcUrlConfig format") + } + } +} + data class ConnectorConfiguration( - val jdbcUrl: String = "", + val jdbcUrl: JdbcUrlConfig = JdbcUrlConfig.EnvVar("JDBC_URL"), val jdbcProperties: Map = emptyMap(), val schemas: List = emptyList(), val tables: List = emptyList(), @@ -36,4 +59,4 @@ data class ConnectorConfiguration( } } } -} \ No newline at end of file +}