Accept jdbcUrl as an environment variable (#32)

* Accept jdbcUrl as an environment variable

* CLI updates

* Update metadata files

ndc-app/src/main/kotlin/io/hasura/ndc/app/services/AgroalDataSourceService.kt

@@ -7,6 +7,7 @@ import io.agroal.api.configuration.supplier.AgroalDataSourceConfigurationSupplie
 import io.agroal.api.security.NamePrincipal
 import io.agroal.api.security.SimplePassword
 import io.hasura.ndc.common.ConnectorConfiguration
+import io.hasura.ndc.common.JdbcUrlConfig
 import io.opentelemetry.instrumentation.annotations.WithSpan
 import io.opentelemetry.instrumentation.jdbc.datasource.OpenTelemetryDataSource
 import io.quarkus.agroal.runtime.AgroalOpenTelemetryWrapper
@@ -174,7 +175,7 @@ class AgroalDataSourceService {
     }
 
     private fun mkAgroalDataSourceConfigurationSupplier(
-        jdbcUrl: String,
+        jdbcUrl: JdbcUrlConfig,
         properties: Map<String, Any>
     ) =
         AgroalDataSourceConfigurationSupplier().metricsEnabled().connectionPoolConfiguration { connectionPool ->
@@ -193,7 +194,10 @@ class AgroalDataSourceService {
                 .multipleAcquisition(config.connectionPoolConfiguration().multipleAcquisition())
                 .enhancedLeakReport(config.connectionPoolConfiguration().enhancedLeakReport())
                 .connectionFactoryConfiguration { connFactory ->
-                    connFactory.jdbcUrl(jdbcUrl)
+                    connFactory.jdbcUrl(when (jdbcUrl) {
+                        is JdbcUrlConfig.Literal -> jdbcUrl.value
+                        is JdbcUrlConfig.EnvVar -> System.getenv(jdbcUrl.variable) ?: throw IllegalArgumentException("Environment variable ${jdbcUrl.variable} not found")
+                    })
                     connFactory.loginTimeout(config.connectionFactoryConfiguration().loginTimeout())
                     // To explain what is going on here:
                     //

ndc-cli/src/main/kotlin/io/hasura/cli/IConfigGenerator.kt

@@ -1,7 +1,8 @@
 package io.hasura.cli
 
 import io.hasura.ndc.common.ConnectorConfiguration
+import io.hasura.ndc.common.JdbcUrlConfig
 
 interface IConfigGenerator {
-    fun getConfig(jdbcUrl: String, schemas: List<String>): ConnectorConfiguration
+    fun getConfig(jdbcUrlConfig: JdbcUrlConfig, schemas: List<String>): ConnectorConfiguration
 }

ndc-cli/src/main/kotlin/io/hasura/cli/MySQLConfigGenerator.kt

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import io.hasura.ndc.common.ColumnSchemaRow
 import io.hasura.ndc.common.ConnectorConfiguration
+import io.hasura.ndc.common.JdbcUrlConfig
 import io.hasura.ndc.common.TableSchemaRow
 import io.hasura.ndc.common.TableType
 import org.jooq.impl.DSL
@@ -12,10 +13,15 @@ object MySQLConfigGenerator : IConfigGenerator {
     private val mapper = jacksonObjectMapper()
 
     override fun getConfig(
-        jdbcUrl: String,
+        jdbcUrl: JdbcUrlConfig,
         schemas: List<String>
     ): ConnectorConfiguration {
-        val ctx = DSL.using(jdbcUrl)
+        val jdbcUrlString = when (jdbcUrl) {
+            is JdbcUrlConfig.Literal -> jdbcUrl.value
+            is JdbcUrlConfig.EnvVar -> System.getenv(jdbcUrl.variable)
+                ?: throw IllegalArgumentException("Environment variable ${jdbcUrl.variable} not found")
+        }
+        val ctx = DSL.using(jdbcUrlString)
 
         //language=MySQL
         val sql = """

ndc-cli/src/main/kotlin/io/hasura/cli/OracleConfigGenerator.kt

@@ -3,6 +3,7 @@ package io.hasura.cli
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import io.hasura.ndc.common.ConnectorConfiguration
+import io.hasura.ndc.common.JdbcUrlConfig
 import io.hasura.ndc.common.TableSchemaRow
 import io.hasura.ndc.common.TableType
 import org.jooq.impl.DSL
@@ -12,10 +13,15 @@ object OracleConfigGenerator : IConfigGenerator {
     private val mapper = jacksonObjectMapper()
 
     override fun getConfig(
-        jdbcUrl: String,
+        jdbcUrl: JdbcUrlConfig,
         schemas: List<String>
     ): ConnectorConfiguration {
-        val ctx = DSL.using(jdbcUrl)
+        val jdbcUrlString = when (jdbcUrl) {
+            is JdbcUrlConfig.Literal -> jdbcUrl.value
+            is JdbcUrlConfig.EnvVar -> System.getenv(jdbcUrl.variable)
+                ?: throw IllegalArgumentException("Environment variable ${jdbcUrl.variable} not found")
+        }
+        val ctx = DSL.using(jdbcUrlString)
 
         //language=Oracle
         val baseTableSql = """

ndc-cli/src/main/kotlin/io/hasura/cli/SnowflakeConfigGenerator.kt

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import io.hasura.ndc.common.ColumnSchemaRow
 import io.hasura.ndc.common.ConnectorConfiguration
+import io.hasura.ndc.common.JdbcUrlConfig
 import io.hasura.ndc.common.TableSchemaRow
 import io.hasura.ndc.common.TableType
 import io.hasura.ndc.ir.ForeignKeyConstraint
@@ -13,11 +14,17 @@ object SnowflakeConfigGenerator : IConfigGenerator {
     private val mapper = jacksonObjectMapper()
 
     override fun getConfig(
-        jdbcUrl: String,
+        jdbcUrl: JdbcUrlConfig,
         schemas: List<String>
     ): ConnectorConfiguration {
+        val jdbcUrlString = when (jdbcUrl) {
+            is JdbcUrlConfig.Literal -> jdbcUrl.value
+            is JdbcUrlConfig.EnvVar -> System.getenv(jdbcUrl.variable)
+                ?: throw IllegalArgumentException("Environment variable ${jdbcUrl.variable} not found")
+        }
+
         // Don't use Arrow memory format so we don't need to --add-opens=java.base/java.nio=ALL-UNNAMED to the JVM
-        val modifiedJdbcUrl = jdbcUrl.find { it == '?' }
+        val modifiedJdbcUrl = jdbcUrlString.find { it == '?' }
             ?.let { "$jdbcUrl&JDBC_QUERY_RESULT_FORMAT=JSON" }
             ?: "$jdbcUrl?JDBC_QUERY_RESULT_FORMAT=JSON"
 

ndc-cli/src/main/kotlin/io/hasura/cli/main.kt

@@ -1,7 +1,9 @@
 package io.hasura.cli
 
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
 import io.hasura.ndc.common.ConnectorConfiguration
+import io.hasura.ndc.common.JdbcUrlConfig
 import picocli.CommandLine
 import picocli.CommandLine.*
 import java.io.File
@@ -34,11 +36,11 @@ class CLI {
     )
     fun update(
         @Parameters(
-            arity = "1",
+            arity = "0..1",
             paramLabel = "<jdbcUrl>",
-            description = ["JDBC URL to connect to the Oracle database"]
+            description = ["JDBC URL to connect to the database (optional)"]
         )
-        jdbcUrl: String,
+        jdbcUrlParam: String?,
         @Option(
             names = ["-o", "--outfile"],
             defaultValue = "configuration.json",
@@ -60,14 +62,37 @@ class CLI {
     ) {
         val file = File(outfile)
 
-        println("Checking for configuration file at ${file.absolutePath}")
-        val existingConfig = file.let {
-            if (it.exists()) {
-                println("Existing configuration file detected")
-                mapper.readValue(it, ConnectorConfiguration::class.java)
-            } else {
-                println("Non-existent or empty configuration file detected")
-                ConnectorConfiguration()
+        // Parse existing configuration
+        val existingConfig = if (file.exists()) {
+            println("Existing configuration file detected at ${file.absolutePath}")
+            try {
+                mapper.readValue<ConnectorConfiguration>(file)
+            } catch (e: Exception) {
+                println("Error reading existing configuration: ${e.message}")
+                null
+            }
+        } else {
+            println("No existing configuration file found at ${file.absolutePath}")
+            null
+        }
+
+        // Determine the JDBC URL configuration
+        val jdbcUrlConfig = when {
+            jdbcUrlParam != null -> {
+                // If jdbcUrlParam is provided, use it as a literal value
+                JdbcUrlConfig.Literal(jdbcUrlParam)
+            }
+            System.getenv("JDBC_URL") != null -> {
+                // If JDBC_URL environment variable is set, use it
+                JdbcUrlConfig.EnvVar("JDBC_URL")
+            }
+            existingConfig?.jdbcUrl != null -> {
+                // If there's an existing config, use its jdbcUrl (which could be either Literal or EnvVar)
+                existingConfig.jdbcUrl
+            }
+            else -> {
+                // If none of the above conditions are met, throw an error
+                throw IllegalArgumentException("No JDBC URL provided and no existing configuration found")
             }
         }
 
@@ -77,18 +102,21 @@ class CLI {
             DatabaseType.SNOWFLAKE -> SnowflakeConfigGenerator
         }
 
-        println("Generating configuration for $database database...")
+        println("Introspecting database...")
         val introspectedConfig = configGenerator.getConfig(
-            jdbcUrl = jdbcUrl,
-            schemas = schemas ?: emptyList()
+            jdbcUrlConfig = jdbcUrlConfig,
+            schemas = schemas ?: existingConfig?.schemas ?: emptyList()
         )
-        val mergedConfigWithNativeQueries = introspectedConfig.copy(
-            nativeQueries = existingConfig.nativeQueries
+
+        val finalConfig = introspectedConfig.copy(
+            jdbcUrl = jdbcUrlConfig,
+            nativeQueries = existingConfig?.nativeQueries ?: emptyMap()
         )
 
         try {
-            println("Writing configuration to ${file.absolutePath}")
-            mapper.writerWithDefaultPrettyPrinter().writeValue(file, mergedConfigWithNativeQueries)
+            println("Writing updated configuration to ${file.absolutePath}")
+            mapper.writerWithDefaultPrettyPrinter().writeValue(file, finalConfig)
+            println("Configuration updated successfully")
         } catch (e: Exception) {
             println("Error writing configuration to file: ${e.message}")
 

ndc-connector-mysql/.hasura-connector/connector-metadata.yaml

@@ -7,9 +7,10 @@ supportedEnvironmentVariables:
 commands:
   update: |
     docker run \
+      -e JDBC_URL \
       -e HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH \
       -v ${HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH}:/app/output \
-      ghcr.io/hasura/ndc-jvm-cli:v0.1.3 update $JDBC_URL \
+      ghcr.io/hasura/ndc-jvm-cli:v0.1.3 update \
         --database MYSQL \
         --schemas $JDBC_SCHEMAS \
         --outfile /app/output/configuration.json

ndc-connector-oracle/.hasura-connector/connector-metadata.yaml

@@ -9,9 +9,10 @@ supportedEnvironmentVariables:
 commands:
   update: |
     docker run \
+      -e JDBC_URL \
       -e HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH \
       -v ${HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH}:/app/output \
-      ghcr.io/hasura/ndc-jvm-cli:v0.1.3 update $JDBC_URL \
+      ghcr.io/hasura/ndc-jvm-cli:v0.1.3 update \
         --database ORACLE \
         --schemas $JDBC_SCHEMAS \
         --outfile /app/output/configuration.json

ndc-connector-snowflake/.hasura-connector/connector-metadata.yaml

@@ -9,9 +9,10 @@ supportedEnvironmentVariables:
 commands:
   update: |
     docker run \
+      -e JDBC_URL \
       -e HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH \
       -v ${HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH}:/app/output \
-      ghcr.io/hasura/ndc-jvm-cli:v0.1.0 update $JDBC_URL \
+      ghcr.io/hasura/ndc-jvm-cli:v0.1.0 update \
         --database SNOWFLAKE \
         --schemas $JDBC_SCHEMAS \
         --outfile /app/output/configuration.json

ndc-ir/src/main/kotlin/io/hasura/ndc/common/ConnectorConfiguration.kt

@@ -2,11 +2,34 @@ package io.hasura.ndc.common
 
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize
+import com.fasterxml.jackson.core.JsonParser
+import com.fasterxml.jackson.databind.DeserializationContext
+import com.fasterxml.jackson.databind.JsonDeserializer
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.annotation.JsonValue
 import java.io.File
 import java.nio.file.Path
 
+@JsonDeserialize(using = JdbcUrlConfigDeserializer::class)
+sealed class JdbcUrlConfig {
+    data class Literal(@JsonValue val value: String) : JdbcUrlConfig()
+    data class EnvVar(val variable: String) : JdbcUrlConfig()
+}
+
+class JdbcUrlConfigDeserializer : JsonDeserializer<JdbcUrlConfig>() {
+    override fun deserialize(p: JsonParser, ctxt: DeserializationContext): JdbcUrlConfig {
+        val node: JsonNode = p.codec.readTree(p)
+        return when {
+            node.isTextual -> JdbcUrlConfig.Literal(node.asText())
+            node.isObject && node.has("variable") -> JdbcUrlConfig.EnvVar(node.get("variable").asText())
+            else -> throw IllegalArgumentException("Invalid JdbcUrlConfig format")
+        }
+    }
+}
+
 data class ConnectorConfiguration(
-    val jdbcUrl: String = "",
+    val jdbcUrl: JdbcUrlConfig = JdbcUrlConfig.EnvVar("JDBC_URL"),
     val jdbcProperties: Map<String, Any> = emptyMap(),
     val schemas: List<String> = emptyList(),
     val tables: List<TableSchemaRow> = emptyList(),
@@ -36,4 +59,4 @@ data class ConnectorConfiguration(
             }
         }
     }
-}
+}