From dc4c4ed909e738ad16d63f0bdaeec7aed8ea5582 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Thu, 25 Jan 2024 03:06:05 +0100 Subject: [PATCH] wip --- .../api/client/ITDeltaSharingClient.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java index fc052021a..55447b1a5 100644 --- a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java +++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java @@ -24,6 +24,7 @@ public class ITDeltaSharingClient implements DatasetComparer, ScalaUtils { private final StorageManagerInitializer storageManagerInitializer; private final String deltaTablePath; + public ITDeltaSharingClient() { this.storageManagerInitializer = new StorageManagerInitializer(); this.deltaTablePath = @@ -59,6 +60,30 @@ void showS3Table1withQueryTableApi() { assertSmallDatasetEquality(ds, expectedData, true, false, false, 500); } + @Test + void showS3IcebergTableWithQueryTableApi() { + var spark = TestSparkSession.newSparkSession(); + registerAnIcebergTable(); + var ds = spark.read().format("deltaSharing").load(deltaTablePath); + var expectedSchema = new StructType(new StructField[] { + new StructField("id", DataType.fromDDL("long"), true, new Metadata(emptyScalaMap())) + }); + var expectedData = spark + .createDataFrame( + List.of( + new MrFoxDeltaTableSchema(0), + new MrFoxDeltaTableSchema(3), + new MrFoxDeltaTableSchema(2), + new MrFoxDeltaTableSchema(1), + new MrFoxDeltaTableSchema(4)), + MrFoxDeltaTableSchema.class) + .toDF(); + + assertEquals(expectedSchema, ds.schema()); + assertEquals(5, ds.count()); + assertSmallDatasetEquality(ds, expectedData, true, false, false, 500); + } + @Test void registerAnIcebergTable() { TableInfo tableInfo = storageManagerInitializer.createIcebergTableWithGlueMetastore();