diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java index 0c9c506569..894a3e82ca 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -19,6 +19,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import java.time.Instant; import java.time.LocalDate; @@ -69,7 +70,8 @@ public class ExprTimestampValue extends AbstractExprValue { */ public ExprTimestampValue(String timestamp) { try { - this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_MICROS) + this.timestamp = LocalDateTime.parse(timestamp, + ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER) .atZone(ZONE) .toInstant(); } catch (DateTimeParseException e) { diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java similarity index 84% rename from elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java rename to core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java index c653c063ac..818eb71375 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java @@ -15,7 +15,7 @@ * */ -package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value; +package com.amazon.opendistroforelasticsearch.sql.data.utils; import static java.time.temporal.ChronoField.DAY_OF_MONTH; import static java.time.temporal.ChronoField.HOUR_OF_DAY; @@ -37,7 +37,7 @@ * Reference org.elasticsearch.common.time.DateFormatters. */ @UtilityClass -public class ElasticsearchDateFormatters { +public class ExprDateFormatters { public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON = new DateTimeFormatterBuilder() @@ -73,7 +73,16 @@ public class ElasticsearchDateFormatters { new DateTimeFormatterBuilder() .append(STRICT_YEAR_MONTH_DAY_FORMATTER) .optionalStart() + + // otherwise use space + .optionalStart() + .appendLiteral(' ') + .optionalEnd() + // optional T + .optionalStart() .appendLiteral('T') + .optionalEnd() + .optionalStart() .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) .optionalStart() @@ -82,13 +91,18 @@ public class ElasticsearchDateFormatters { .optionalStart() .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) + + // optional millis with dot .optionalStart() .appendFraction(NANO_OF_SECOND, 1, 9, true) .optionalEnd() + + // otherwise optional millis use with comma .optionalStart() .appendLiteral(',') .appendFraction(NANO_OF_SECOND, 1, 9, false) .optionalEnd() + .optionalEnd() .optionalEnd() .optionalStart() @@ -104,4 +118,11 @@ public class ElasticsearchDateFormatters { public static final DateTimeFormatter SQL_LITERAL_DATE_TIME_FORMAT = DateTimeFormatter .ofPattern("yyyy-MM-dd HH:mm:ss"); + + public static final DateTimeFormatter TOLERANT_PARSER_DATE_TIME_FORMATTER = + new DateTimeFormatterBuilder() + .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER) + .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT) + .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER) + .toFormatter(); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java index 294c21ec52..071258ee21 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java @@ -113,9 +113,9 @@ public void timeInUnsupportedFormat() { public void timestampInUnsupportedFormat() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimestampValue("2020-07-07T01:01:01Z")); + () -> new ExprTimestampValue("2020-07-07T1:01:01Z")); assertEquals( - "timestamp:2020-07-07T01:01:01Z in unsupported format, " + "timestamp:2020-07-07T1:01:01Z in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } @@ -239,9 +239,9 @@ public void datetimeWithVariableMicroPrecision() { public void timestampOverMaxMicroPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimestampValue("2020-07-07 01:01:01.1234567")); + () -> new ExprTimestampValue("2020-07-07 01:01:01.12345678910")); assertEquals( - "timestamp:2020-07-07 01:01:01.1234567 in unsupported format, " + "timestamp:2020-07-07 01:01:01.12345678910 in unsupported format, " + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", exception.getMessage()); } diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java index 2fbae486d6..b0f768ce94 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java @@ -36,9 +36,6 @@ import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_IP; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.SQL_LITERAL_DATE_TIME_FORMAT; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.STRICT_DATE_OPTIONAL_TIME_FORMATTER; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.STRICT_HOUR_MINUTE_SECOND_FORMATTER; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; @@ -57,6 +54,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.utils.Content; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.utils.ElasticsearchJsonContent; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.utils.ObjectContent; @@ -64,8 +62,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -87,13 +83,6 @@ public class ElasticsearchExprValueFactory { @Setter private Map typeMapping; - private static final DateTimeFormatter DATE_TIME_FORMATTER = - new DateTimeFormatterBuilder() - .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT) - .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER) - .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER) - .toFormatter(); - private static final String TOP_PATH = ""; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -185,7 +174,8 @@ private ExprValue constructTimestamp(String value) { try { return new ExprTimestampValue( // Using Elasticsearch DateFormatters for now. - DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant()); + DateFormatters.from(ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER + .parse(value)).toInstant()); } catch (DateTimeParseException e) { throw new IllegalStateException( String.format( diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java index badf6afb57..a40921a64c 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java @@ -190,6 +190,12 @@ public void constructDate() { assertEquals( new ExprTimestampValue("2015-01-01 12:10:30"), tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")); + assertEquals( + new ExprTimestampValue("2020-08-17T19:44:00.100500"), + tupleValue("{\"timestampV\":\"2020-08-17T19:44:00.100500\"}").get("timestampV")); + assertEquals( + new ExprTimestampValue("2020-08-17 19:44:00.100500"), + tupleValue("{\"timestampV\":\"2020-08-17 19:44:00.100500\"}").get("timestampV")); assertEquals( new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), tupleValue("{\"timestampV\":1420070400001}").get("timestampV")); @@ -221,9 +227,10 @@ public void constructDate() { public void constructDateFromUnsupportedFormatThrowException() { IllegalStateException exception = assertThrows( - IllegalStateException.class, () -> tupleValue("{\"timestampV\":\"2015-01-01 12:10\"}")); + IllegalStateException.class, () -> + tupleValue("{\"timestampV\":\"2015-01-01 1:10:10\"}")); assertEquals( - "Construct ExprTimestampValue from \"2015-01-01 12:10\" failed, " + "Construct ExprTimestampValue from \"2015-01-01 1:10:10\" failed, " + "unsupported date format.", exception.getMessage()); }