From c589fe6f3c7b78a570b82a79e3d7d7349e9eeb45 Mon Sep 17 00:00:00 2001
From: Aleksandar Milosevic <aleksandarmilosevic98@gmail.com>
Date: Mon, 29 Jan 2024 22:35:18 +0100
Subject: [PATCH] fixed catalog loading

---
 .../io/whitefox/api/client/ITDeltaSharingClient.java  |  4 +++-
 .../test/java/io/whitefox/api/utils/S3TestConfig.java | 11 +++++++++--
 .../whitefox/api/utils/StorageManagerInitializer.java |  9 +++++----
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java
index ab595325c..7a923b181 100644
--- a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java
+++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java
@@ -44,7 +44,7 @@ void showS3IcebergTable1withQueryTableApi() {
     storageManagerInitializer.createIcebergTableWithGlueMetastore();
     var ds = spark.read().format("deltaSharing").load(icebergTablePath);
     var expectedSchema = new StructType(new StructField[] {
-      new StructField("id", DataType.fromDDL("long"), true, new Metadata(emptyScalaMap()))
+      new StructField("id", DataType.fromDDL("long"), false, new Metadata(emptyScalaMap()))
     });
     var expectedData = spark
         .createDataFrame(
@@ -70,6 +70,8 @@ void showS3Table1withQueryTableApi() {
     var expectedSchema = new StructType(new StructField[] {
       new StructField("id", DataType.fromDDL("long"), true, new Metadata(emptyScalaMap()))
     });
+
+    ds.show();
     var expectedData = spark
         .createDataFrame(
             List.of(
diff --git a/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java b/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java
index 3f1a15f85..72b0a59ce 100644
--- a/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java
+++ b/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java
@@ -4,6 +4,7 @@ public class S3TestConfig {
   private final String region;
   private final String accessKey;
   private final String secretKey;
+  private final String glueCatalogId;
 
   public String getRegion() {
     return region;
@@ -17,16 +18,22 @@ public String getSecretKey() {
     return secretKey;
   }
 
-  public S3TestConfig(String region, String accessKey, String secretKey) {
+  public String getGlueCatalogId() {
+    return glueCatalogId;
+  }
+
+  public S3TestConfig(String region, String accessKey, String secretKey, String glueCatalogId) {
     this.region = region;
     this.accessKey = accessKey;
     this.secretKey = secretKey;
+    this.glueCatalogId = glueCatalogId;
   }
 
   public static S3TestConfig loadFromEnv() {
     return new S3TestConfig(
         System.getenv().get("WHITEFOX_TEST_AWS_REGION"),
         System.getenv().get("WHITEFOX_TEST_AWS_ACCESS_KEY_ID"),
-        System.getenv().get("WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY"));
+        System.getenv().get("WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY"),
+        System.getenv().get("WHITEFOX_TEST_GLUE_CATALOG_ID"));
   }
 }
diff --git a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java
index 2a91d6991..8824003bd 100644
--- a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java
+++ b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java
@@ -62,13 +62,14 @@ public TableInfo createIcebergTableWithGlueMetastore() {
     var shareRequest = createShareRequest();
     ignoreConflict(() -> schemaV1Api.createSchema(shareRequest.getName(), schemaRequest));
     var createTableRequest = createIcebergTableRequest();
+    ApiUtils.recoverConflictLazy(
+        () -> tableV1Api.createTableInProvider(provider.getName(), createTableRequest),
+        () -> tableV1Api.describeTableInProvider(provider.getName(), createTableRequest.getName()));
     ignoreConflict(() -> schemaV1Api.addTableToSchema(
         shareRequest.getName(),
         schemaRequest,
         addTableToSchemaRequest(providerRequest.getName(), createTableRequest.getName())));
-    return ApiUtils.recoverConflictLazy(
-        () -> tableV1Api.createTableInProvider(provider.getName(), createTableRequest),
-        () -> tableV1Api.describeTableInProvider(provider.getName(), createTableRequest.getName()));
+    return tableV1Api.describeTableInProvider(provider.getName(), createTableRequest.getName());
   }
 
   private String createSchemaRequest(TableFormat tableFormat) {
@@ -129,7 +130,7 @@ private CreateMetastore createMetastoreRequest(
         .type(type)
         .skipValidation(true)
         .properties(new MetastoreProperties(new GlueProperties()
-            .catalogId("catalogId") // TODO
+            .catalogId(s3TestConfig.getGlueCatalogId())
             .credentials(new SimpleAwsCredentials()
                 .region(s3TestConfig.getRegion())
                 .awsAccessKeyId(s3TestConfig.getAccessKey())