Skip to content

Commit

Permalink
Return read version in ReadTableResult (#198)
Browse files Browse the repository at this point in the history
Add lombok because pojo refactor was too painful
  • Loading branch information
agilelab-tmnd1991 authored Jan 29, 2024
1 parent 9c58f42 commit 79c1eab
Show file tree
Hide file tree
Showing 10 changed files with 60 additions and 141 deletions.
1 change: 1 addition & 0 deletions buildSrc/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,5 @@ dependencies {
implementation("org.openapi.generator:org.openapi.generator.gradle.plugin:6.6.0")
implementation("com.diffplug.spotless:spotless-plugin-gradle:6.25.0")
implementation("com.palantir.gradle.gitversion:gradle-git-version:3.0.0")
implementation("io.freefair.lombok:io.freefair.lombok.gradle.plugin:8.4")
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ plugins {
id("com.palantir.git-version")
id("org.openapi.generator")
id("com.diffplug.spotless")
id("io.freefair.lombok")
}
// Projects should use Maven Central for external dependencies
repositories {
Expand Down
2 changes: 2 additions & 0 deletions lombok.config
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
lombok.accessors.fluent=true
lombok.addLombokGeneratedAnnotation = true
6 changes: 0 additions & 6 deletions protocol/delta-sharing-protocol-api.yml
Original file line number Diff line number Diff line change
Expand Up @@ -413,12 +413,6 @@ paths:
description: 'Named table'
schema:
type: string
- in: query
name: startingTimestamp
required: false
description: 'Starting Timestamp ISO8601 format, in the UTC timezone'
schema:
type: string
- in: header
name: delta-sharing-capabilities
required: false
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,49 +164,35 @@ share, schema, parseToken(pageToken), Optional.ofNullable(maxResults)),
exceptionToResponse);
}

/***
* Example:
* TableQueryResponse:
* type: string
* example: |
* {"protocol":{"minReaderVersion":1}}
* {"metaData":{"id":"f8d5c169-3d01-4ca3-ad9e-7dc3355aedb2","format":{"provider":"parquet"},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"eventTime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date"]}}
* {"file":{"url":"https://<s3-bucket-name>.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-8b0086f2-7b27-4935-ac5a-8ed6215a6640.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=900&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=97b6762cfd8e4d7e94b9d707eff3faf266974f6e7030095c1d4a66350cfd892e","id":"8b0086f2-7b27-4935-ac5a-8ed6215a6640","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"nullCount\":{\"eventTime\":0}}"}}
* {"file":{"url":"https://<s3-bucket-name>.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-591723a8-6a27-4240-a90e-57426f4736d2.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=899&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=0f7acecba5df7652457164533a58004936586186c56425d9d53c52db574f6b62","id":"591723a8-6a27-4240-a90e-57426f4736d2","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"nullCount\":{\"eventTime\":0}}"}}
*
*
* @param share
* @param schema
* @param table
* @param queryRequest
* @param startingTimestamp
* @return
/**
* <pre>
* {"protocol":{"minReaderVersion":1}}
* {"metaData":{"id":"f8d5c169-3d01-4ca3-ad9e-7dc3355aedb2","format":{"provider":"parquet"},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"eventTime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date"]}}
* {"file":{"url":"https://<s3-bucket-name>.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-8b0086f2-7b27-4935-ac5a-8ed6215a6640.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=900&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=97b6762cfd8e4d7e94b9d707eff3faf266974f6e7030095c1d4a66350cfd892e","id":"8b0086f2-7b27-4935-ac5a-8ed6215a6640","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"nullCount\":{\"eventTime\":0}}"}}
* {"file":{"url":"https://<s3-bucket-name>.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-591723a8-6a27-4240-a90e-57426f4736d2.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=899&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=0f7acecba5df7652457164533a58004936586186c56425d9d53c52db574f6b62","id":"591723a8-6a27-4240-a90e-57426f4736d2","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"nullCount\":{\"eventTime\":0}}"}}
* </pre>
*/
@Override
public Response queryTable(
String share,
String schema,
String table,
QueryRequest queryRequest,
String startingTimestamp,
String deltaSharingCapabilities) {
return wrapExceptions(
() -> optionalToNotFound(
deltaSharesService.getTableVersion(share, schema, table, startingTimestamp),
version -> Response.ok(
tableQueryResponseSerializer.serialize(
DeltaMappers.readTableResult2api(deltaSharesService.queryTable(
share,
schema,
table,
DeltaMappers.api2ReadTableRequest(queryRequest)))),
ndjsonMediaType)
.header(DELTA_TABLE_VERSION_HEADER, version)
.header(
DELTA_SHARE_CAPABILITIES_HEADER,
getResponseFormatHeader(
DeltaMappers.toHeaderCapabilitiesMap(deltaSharingCapabilities)))
.build()),
() -> {
var readResult = deltaSharesService.queryTable(
share, schema, table, DeltaMappers.api2ReadTableRequest(queryRequest));
var serializedReadResult =
tableQueryResponseSerializer.serialize(DeltaMappers.readTableResult2api(readResult));
return Response.ok(serializedReadResult, ndjsonMediaType)
.header(DELTA_TABLE_VERSION_HEADER, readResult.version())
.header(
DELTA_SHARE_CAPABILITIES_HEADER,
getResponseFormatHeader(
DeltaMappers.toHeaderCapabilitiesMap(deltaSharingCapabilities)))
.build();
},
exceptionToResponse);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,23 @@ public void getTableVersionBadTimestamp() {
.statusCode(502);
}

@DisabledOnOs(OS.WINDOWS)
@Test
public void queryNotExistingTable() throws IOException {
given()
.when()
.filter(deltaFilter)
.body("{}")
.header(new Header("Content-Type", "application/json"))
.post(
"delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query",
"name",
"default",
"tableThatDoesNotExist")
.then()
.statusCode(404);
}

@DisabledOnOs(OS.WINDOWS)
@Test
public void queryTableCurrentVersion() throws IOException {
Expand All @@ -367,6 +384,7 @@ public void queryTableCurrentVersion() throws IOException {
"table1")
.then()
.statusCode(200)
.header("Delta-Table-Version", "0")
.extract()
.body()
.asString()
Expand Down Expand Up @@ -410,6 +428,7 @@ public void queryTableByVersion() throws IOException {
"table1")
.then()
.statusCode(200)
.header("Delta-Table-Version", "0")
.extract()
.body()
.asString()
Expand Down Expand Up @@ -453,6 +472,7 @@ public void queryTableByTs() throws IOException {
"table-with-history")
.then()
.statusCode(200)
.header("Delta-Table-Version", "0")
.extract()
.body()
.asString()
Expand Down
55 changes: 6 additions & 49 deletions server/core/src/main/java/io/whitefox/core/ReadTableResult.java
Original file line number Diff line number Diff line change
@@ -1,55 +1,12 @@
package io.whitefox.core;

import io.whitefox.annotations.SkipCoverageGenerated;
import java.util.List;
import java.util.Objects;
import lombok.Value;

@Value
public class ReadTableResult {
private final Protocol protocol;
private final Metadata metadata;
private final List<TableFile> files;

public ReadTableResult(Protocol protocol, Metadata metadata, List<TableFile> files) {
this.protocol = protocol;
this.metadata = metadata;
this.files = files;
}

@Override
@SkipCoverageGenerated
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ReadTableResult that = (ReadTableResult) o;
return Objects.equals(protocol, that.protocol)
&& Objects.equals(metadata, that.metadata)
&& Objects.equals(files, that.files);
}

@Override
@SkipCoverageGenerated
public int hashCode() {
return Objects.hash(protocol, metadata, files);
}

@Override
@SkipCoverageGenerated
public String toString() {
return "QueryTableResult{" + "protocol="
+ protocol + ", metadata="
+ metadata + ", files="
+ files + '}';
}

public Protocol protocol() {
return protocol;
}

public Metadata metadata() {
return metadata;
}

public List<TableFile> files() {
return files;
}
Protocol protocol;
Metadata metadata;
List<TableFile> files;
long version;
}
Original file line number Diff line number Diff line change
@@ -1,56 +1,12 @@
package io.whitefox.core;

import io.whitefox.annotations.SkipCoverageGenerated;
import java.util.List;
import java.util.Objects;
import lombok.Value;

@Value
public class ReadTableResultToBeSigned {
private final Protocol protocol;
private final Metadata metadata;
private final List<TableFileToBeSigned> other;

public ReadTableResultToBeSigned(
Protocol protocol, Metadata metadata, List<TableFileToBeSigned> other) {
this.protocol = protocol;
this.metadata = metadata;
this.other = other;
}

@Override
@SkipCoverageGenerated
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ReadTableResultToBeSigned that = (ReadTableResultToBeSigned) o;
return Objects.equals(protocol, that.protocol)
&& Objects.equals(metadata, that.metadata)
&& Objects.equals(other, that.other);
}

@Override
@SkipCoverageGenerated
public int hashCode() {
return Objects.hash(protocol, metadata, other);
}

@Override
@SkipCoverageGenerated
public String toString() {
return "QueryTableResult{" + "protocol="
+ protocol + ", metadata="
+ metadata + ", other="
+ other + '}';
}

public Protocol protocol() {
return protocol;
}

public Metadata metadata() {
return metadata;
}

public List<TableFileToBeSigned> other() {
return other;
}
Protocol protocol;
Metadata metadata;
List<TableFileToBeSigned> other;
long version;
}
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,8 @@ public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) {
snapshot.getMetadata().getCreatedTime(),
f.getStats(),
f.getPartitionValues()))
.collect(Collectors.toList()));
.collect(Collectors.toList()),
snapshot.getVersion());
}

private Optional<Snapshot> getSnapshot(Optional<String> startingTimestamp) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ public ReadTableResult queryTable(
readTableResultToBeSigned.metadata(),
readTableResultToBeSigned.other().stream()
.map(fileSigner::sign)
.collect(Collectors.toList()));
.collect(Collectors.toList()),
readTableResultToBeSigned.version());
}
}

0 comments on commit 79c1eab

Please sign in to comment.