fixed catalog loading

agile-lab-dev · Jan 29, 2024 · c589fe6 · c589fe6
1 parent d2878e6
commit c589fe6
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 7 deletions.
diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java
@@ -44,7 +44,7 @@ void showS3IcebergTable1withQueryTableApi() {
     storageManagerInitializer.createIcebergTableWithGlueMetastore();
     var ds = spark.read().format("deltaSharing").load(icebergTablePath);
     var expectedSchema = new StructType(new StructField[] {
-      new StructField("id", DataType.fromDDL("long"), true, new Metadata(emptyScalaMap()))
+      new StructField("id", DataType.fromDDL("long"), false, new Metadata(emptyScalaMap()))
     });
     var expectedData = spark
         .createDataFrame(
@@ -70,6 +70,8 @@ void showS3Table1withQueryTableApi() {
     var expectedSchema = new StructType(new StructField[] {
       new StructField("id", DataType.fromDDL("long"), true, new Metadata(emptyScalaMap()))
     });
+
+    ds.show();
     var expectedData = spark
         .createDataFrame(
             List.of(

diff --git a/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java b/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java
@@ -4,6 +4,7 @@ public class S3TestConfig {
   private final String region;
   private final String accessKey;
   private final String secretKey;
+  private final String glueCatalogId;
 
   public String getRegion() {
     return region;
@@ -17,16 +18,22 @@ public String getSecretKey() {
     return secretKey;
   }
 
-  public S3TestConfig(String region, String accessKey, String secretKey) {
+  public String getGlueCatalogId() {
+    return glueCatalogId;
+  }
+
+  public S3TestConfig(String region, String accessKey, String secretKey, String glueCatalogId) {
     this.region = region;
     this.accessKey = accessKey;
     this.secretKey = secretKey;
+    this.glueCatalogId = glueCatalogId;
   }
 
   public static S3TestConfig loadFromEnv() {
     return new S3TestConfig(
         System.getenv().get("WHITEFOX_TEST_AWS_REGION"),
         System.getenv().get("WHITEFOX_TEST_AWS_ACCESS_KEY_ID"),
-        System.getenv().get("WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY"));
+        System.getenv().get("WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY"),
+        System.getenv().get("WHITEFOX_TEST_GLUE_CATALOG_ID"));
   }
 }
diff --git a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java
@@ -62,13 +62,14 @@ public TableInfo createIcebergTableWithGlueMetastore() {
     var shareRequest = createShareRequest();
     ignoreConflict(() -> schemaV1Api.createSchema(shareRequest.getName(), schemaRequest));
     var createTableRequest = createIcebergTableRequest();
+    ApiUtils.recoverConflictLazy(
+        () -> tableV1Api.createTableInProvider(provider.getName(), createTableRequest),
+        () -> tableV1Api.describeTableInProvider(provider.getName(), createTableRequest.getName()));
     ignoreConflict(() -> schemaV1Api.addTableToSchema(
         shareRequest.getName(),
         schemaRequest,
         addTableToSchemaRequest(providerRequest.getName(), createTableRequest.getName())));
-    return ApiUtils.recoverConflictLazy(
-        () -> tableV1Api.createTableInProvider(provider.getName(), createTableRequest),
-        () -> tableV1Api.describeTableInProvider(provider.getName(), createTableRequest.getName()));
+    return tableV1Api.describeTableInProvider(provider.getName(), createTableRequest.getName());
   }
 
   private String createSchemaRequest(TableFormat tableFormat) {
@@ -129,7 +130,7 @@ private CreateMetastore createMetastoreRequest(
         .type(type)
         .skipValidation(true)
         .properties(new MetastoreProperties(new GlueProperties()
-            .catalogId("catalogId") // TODO
+            .catalogId(s3TestConfig.getGlueCatalogId())
             .credentials(new SimpleAwsCredentials()
                 .region(s3TestConfig.getRegion())
                 .awsAccessKeyId(s3TestConfig.getAccessKey())