Added support for Snowflake.

hasura · Sep 19, 2024 · 15653a4 · 15653a4
1 parent b7a21ee
commit 15653a4
Show file tree

Hide file tree

Showing 10 changed files with 299 additions and 2 deletions.
diff --git a/adapters/file/.env b/adapters/file/.env
@@ -5,3 +5,7 @@ OTEL_LOGS_EXPORTER=console
 OTEL_LOG_LEVEL=trace
 LOG4J_CONFIGURATION_FILE=../../calcite-rs-jni/target/classes/log4j2.xml
 DATA_FOLDER=./resources/test/sales
+OTEL_EXPORTER_OTLP_ENDPOINT="http://local.hasura.dev:4317"
+OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://local.hasura.dev:4317"
+OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="http://local.hasura.dev:4317"
+OTEL_SERVICE_NAME=app_calcite
diff --git a/adapters/snowflake/.env b/adapters/snowflake/.env
@@ -0,0 +1,6 @@
+JAR_DEPENDENCY_FOLDER=../../calcite-rs-jni/target/dependency
+CALCITE_JAR=../../calcite-rs-jni/target/calcite-rs-jni-1.0-SNAPSHOT.jar
+RUST_LOG=info
+OTEL_LOGS_EXPORTER=console
+OTEL_LOG_LEVEL=trace
+LOG4J_CONFIGURATION_FILE=../../calcite-rs-jni/target/classes/log4j2.xml
diff --git a/adapters/snowflake/.env.local b/adapters/snowflake/.env.local
@@ -0,0 +1,4 @@
+RUST_LOG=info
+OTEL_LOGS_EXPORTER=console
+OTEL_LOG_LEVEL=trace
+OTEL_EXPORTER_OTLP_ENDPOINT=http://local.hasura.dev:4317
diff --git a/adapters/snowflake/dev.local.configuration.json b/adapters/snowflake/dev.local.configuration.json
@@ -0,0 +1,90 @@
+{
+  "version": "5",
+  "$schema": "schema.json",
+  "model": {
+    "version": "1.0",
+    "defaultSchema": "db2",
+    "schemas": [
+      {
+        "type": "jdbc",
+        "name": "db2",
+        "jdbcUser": "db2inst1",
+        "jdbcPassword": "mypassword",
+        "jdbcUrl": "jdbc:db2://192.168.86.233:50000/testdb"
+      }
+    ]
+  },
+  "model_file_path": "./model.json",
+  "fixes": true,
+  "metadata": {
+    "PROJECTS": {
+      "schema": "db2",
+      "name": "PROJECTS",
+      "columns": {
+        "PROJECTID": {
+          "name": "PROJECTID",
+          "scalarType": "INTEGER",
+          "nullable": false
+        },
+        "PROJECTNAME": {
+          "name": "PROJECTNAME",
+          "scalarType": "VARCHAR",
+          "nullable": true
+        },
+        "DEPARTMENTID": {
+          "name": "DEPARTMENTID",
+          "scalarType": "INTEGER",
+          "nullable": true
+        }
+      },
+      "primaryKeys": [],
+      "exportedKeys": []
+    },
+    "EMPLOYEES": {
+      "schema": "db2",
+      "name": "EMPLOYEES",
+      "columns": {
+        "FIRSTNAME": {
+          "name": "FIRSTNAME",
+          "scalarType": "VARCHAR",
+          "nullable": true
+        },
+        "LASTNAME": {
+          "name": "LASTNAME",
+          "scalarType": "VARCHAR",
+          "nullable": true
+        },
+        "EMPLOYEEID": {
+          "name": "EMPLOYEEID",
+          "scalarType": "INTEGER",
+          "nullable": false
+        },
+        "DEPARTMENTID": {
+          "name": "DEPARTMENTID",
+          "scalarType": "INTEGER",
+          "nullable": true
+        }
+      },
+      "primaryKeys": [],
+      "exportedKeys": []
+    },
+    "DEPARTMENTS": {
+      "schema": "db2",
+      "name": "DEPARTMENTS",
+      "columns": {
+        "DEPARTMENTNAME": {
+          "name": "DEPARTMENTNAME",
+          "scalarType": "VARCHAR",
+          "nullable": true
+        },
+        "DEPARTMENTID": {
+          "name": "DEPARTMENTID",
+          "scalarType": "INTEGER",
+          "nullable": false
+        }
+      },
+      "primaryKeys": [],
+      "exportedKeys": []
+    }
+  }
+}
diff --git a/adapters/snowflake/docker-compose.yaml b/adapters/snowflake/docker-compose.yaml
@@ -0,0 +1,31 @@
+version: '3.8'
+
+services:
+  trino:
+    image: trinodb/trino:latest
+    ports:
+      - "8080:8080"
+    environment:
+      - JAVA_TOOL_OPTIONS=-Duser.timezone=UTC
+    depends_on:
+      - postgres
+    volumes:
+      - trino_catalog:/etc/trino/catalog
+      - ./config:/etc/trino
+      - ./var/trino:/var/trino
+    command: /usr/lib/trino/bin/run-trino
+
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_USER: trino
+      POSTGRES_PASSWORD: trino
+      POSTGRES_DB: postgres
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+volumes:
+  trino_catalog:
+  postgres_data:
diff --git a/adapters/snowflake/model.json b/adapters/snowflake/model.json
@@ -0,0 +1,16 @@
+{
+  "version": "1.0",
+  "defaultSchema": "TEST",
+  "schemas": [
+    {
+      "name": "TEST",
+      "type": "jdbc",
+      "jdbcDriver": "net.snowflake.client.jdbc.SnowflakeDriver",
+      "jdbcUrl": "jdbc:snowflake://ukpyrhb-xpb27951.snowflakecomputing.com/?db=SNOWFLAKE_SAMPLE_DATA&schema=TPCH_SF1&warehouse=COMPUTE_WH&role=accountadmin&user=ken&password=14May1959#",
+      "jdbcSchema": "TPCH_SF1",
+      "jdbcCatalog": "SNOWFLAKE_SAMPLE_DATA",
+      "jdbcUser": "ken",
+      "jdbcPassword": "14May1959#"
+    }
+  ]
+}
diff --git a/adapters/snowflake/schema.json b/adapters/snowflake/schema.json
@@ -0,0 +1,133 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "ParsedConfiguration",
+  "type": "object",
+  "required": [
+    "version"
+  ],
+  "properties": {
+    "version": {
+      "$ref": "#/definitions/Version"
+    },
+    "$schema": {
+      "type": "string"
+    },
+    "model": {
+      "type": "object",
+      "properties": {
+        "version": {
+          "type": "string"
+        },
+        "defaultSchema": {
+          "type": "string"
+        },
+        "schemas": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "type": "string"
+              },
+              "name": {
+                "type": "string"
+              },
+              "jdbcUser": {
+                "type": "string"
+              },
+              "jdbcPassword": {
+                "type": "string"
+              },
+              "jdbcUrl": {
+                "type": "string"
+              },
+              "sqlDialectFactory": {
+                "type": "string"
+              },
+              "jdbcCatalog": {
+                "type": "string"
+              },
+              "jdbcSchema": {},
+              "factory": {
+                "type": "string"
+              },
+              "operand": {
+                "type": "object",
+                "properties": {
+                  "directory": {
+                    "type": "string"
+                  },
+                  "host": {
+                    "type": "string"
+                  },
+                  "port": {
+                    "type": "number"
+                  },
+                  "database": {
+                    "type": "string"
+                  },
+                  "password": {
+                    "type": "string"
+                  }
+                }
+              },
+              "tables": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "name": {
+                      "type": "string"
+                    },
+                    "factory": {
+                      "type": "string"
+                    },
+                    "operand": {
+                      "type": "object",
+                      "properties": {
+                        "dataFormat": {
+                          "type": "string"
+                        },
+                        "fields": {
+                          "type": "array",
+                          "items": {
+                            "type": "object",
+                            "properties": {
+                              "name": {
+                                "type": "string"
+                              },
+                              "type": {
+                                "type": "string"
+                              },
+                              "mapping": {}
+                            },
+                            "required": [
+                              "name",
+                              "type",
+                              "mapping"
+                            ]
+                          }
+                        },
+                        "keyDelimiter": {
+                          "type": "string"
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "definitions": {
+    "Version": {
+      "type": "string",
+      "enum": [
+        "5"
+      ]
+    }
+  }
+}
diff --git a/calcite-rs-jni/pom.xml b/calcite-rs-jni/pom.xml
@@ -401,6 +401,18 @@
             <artifactId>h2</artifactId>
             <version>2.2.220</version>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/net.snowflake/snowflake-jdbc -->
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-jdbc</artifactId>
+            <version>3.19.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.databricks/databricks-jdbc -->
+        <dependency>
+            <groupId>com.databricks</groupId>
+            <artifactId>databricks-jdbc</artifactId>
+            <version>2.6.40</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-jdbc</artifactId>

diff --git a/crates/calcite-schema/src/jvm.rs b/crates/calcite-schema/src/jvm.rs
@@ -109,7 +109,7 @@ pub fn init_jvm(calcite_configuration: &ParsedConfiguration) {
         let mut jvm_args = InitArgsBuilder::new()
             .version(JNIVersion::V8)
             .option(format!("-Dlog4j2.debug={}", log4j2_debug))
-            .option("--add-opens=java.base/java.nio=ALL-UNNAMED")
+            .option("--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED")
             .option("-Dotel.java.global-autoconfigure.enabled=true")
             .option(format!("-Dlog4j.configurationFile={}", log4j_configuration_file));
         if !otel_exporter_otlp_traces_endpoint.is_empty() {

diff --git a/crates/connectors/ndc-calcite/src/lib.rs b/crates/connectors/ndc-calcite/src/lib.rs
@@ -35,7 +35,8 @@
 //! - Vertica
 //! - Sybase
 //! - StarRocks
-//! - Snowflake
+//! - Snowflake (tested)
+//! - Databricks
 //! - Presto
 //! - Trino (tested)
 //! - Phoenix