From cc95c27cd3965776d96df5fbb7937ab159715a35 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 25 Sep 2015 15:25:29 -0700 Subject: [PATCH 01/83] Ignore if the partitions have no corresponfing hdfs valid path. --- .../java/com/facebook/presto/hive/util/HiveFileIterator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java index 160f1dc0e964..077711b17982 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java @@ -33,7 +33,6 @@ import java.util.Properties; import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILE_NOT_FOUND; import static java.util.Objects.requireNonNull; public class HiveFileIterator @@ -94,7 +93,8 @@ protected LocatedFileStatus computeNext() return endOfData(); } catch (FileNotFoundException e) { - throw new PrestoException(HIVE_FILE_NOT_FOUND, "Partition location does not exist: " + path); + // We are okay if the path does not exist. + return endOfData(); } catch (IOException e) { throw new PrestoException(HIVE_FILESYSTEM_ERROR, e); From bce8187ec60304f2f30262ab710b571885b3bdb9 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Thu, 1 Oct 2015 10:40:07 -0700 Subject: [PATCH 02/83] Append nulls for missing values in Parquet. Parquet only calls converts for which it found the values. The missing values are not reported. The BlockBuilder must be appended with nulls for the missing values based on fieldIndex of the currently read value by Parquet. --- .../hive/parquet/ParquetHiveRecordCursor.java | 71 +++++++++++++------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java index 8e9cf0290ff8..c7d8c0c7901f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java @@ -446,7 +446,7 @@ public PrestoReadSupport(boolean useParquetColumnNames, List c converters.add(new ParquetPrimitiveColumnConverter(i)); } else { - converters.add(new ParquetColumnConverter(createGroupConverter(types[i], parquetType.getName(), parquetType), i)); + converters.add(new ParquetColumnConverter(createGroupConverter(types[i], parquetType.getName(), parquetType, i), i)); } } } @@ -674,25 +674,25 @@ private abstract static class GroupedConverter public abstract Block getBlock(); } - private static BlockConverter createConverter(Type prestoType, String columnName, parquet.schema.Type parquetType) + private static BlockConverter createConverter(Type prestoType, String columnName, parquet.schema.Type parquetType, int fieldIndex) { if (parquetType.isPrimitive()) { - return new ParquetPrimitiveConverter(prestoType); + return new ParquetPrimitiveConverter(prestoType, fieldIndex); } - return createGroupConverter(prestoType, columnName, parquetType); + return createGroupConverter(prestoType, columnName, parquetType, fieldIndex); } - private static GroupedConverter createGroupConverter(Type prestoType, String columnName, parquet.schema.Type parquetType) + private static GroupedConverter createGroupConverter(Type prestoType, String columnName, parquet.schema.Type parquetType, int fieldIndex) { GroupType groupType = parquetType.asGroupType(); switch (prestoType.getTypeSignature().getBase()) { case ARRAY: - return new ParquetListConverter(prestoType, columnName, groupType); + return new ParquetListConverter(prestoType, columnName, groupType, fieldIndex); case MAP: - return new ParquetMapConverter(prestoType, columnName, groupType); + return new ParquetMapConverter(prestoType, columnName, groupType, fieldIndex); case ROW: - return new ParquetStructConverter(prestoType, columnName, groupType); + return new ParquetStructConverter(prestoType, columnName, groupType, fieldIndex); default: throw new IllegalArgumentException("Column " + columnName + " type " + parquetType.getOriginalType() + " not supported"); } @@ -705,13 +705,14 @@ private static class ParquetStructConverter private static final int NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD = 32768; private final Type rowType; + private final int fieldIndex; private final List converters; private BlockBuilder builder; private BlockBuilder nullBuilder; // used internally when builder is set to null private BlockBuilder currentEntryBuilder; - public ParquetStructConverter(Type prestoType, String columnName, GroupType entryType) + public ParquetStructConverter(Type prestoType, String columnName, GroupType entryType, int fieldIndex) { checkArgument(ROW.equals(prestoType.getTypeSignature().getBase())); List prestoTypeParameters = prestoType.getTypeParameters(); @@ -719,11 +720,12 @@ public ParquetStructConverter(Type prestoType, String columnName, GroupType entr checkArgument(prestoTypeParameters.size() == fieldTypes.size()); this.rowType = prestoType; + this.fieldIndex = fieldIndex; ImmutableList.Builder converters = ImmutableList.builder(); for (int i = 0; i < prestoTypeParameters.size(); i++) { parquet.schema.Type fieldType = fieldTypes.get(i); - converters.add(createConverter(prestoTypeParameters.get(i), columnName + "." + fieldType.getName(), fieldType)); + converters.add(createConverter(prestoTypeParameters.get(i), columnName + "." + fieldType.getName(), fieldType, i)); } this.converters = converters.build(); } @@ -750,6 +752,9 @@ public void start() currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { + while (builder.getPositionCount() < fieldIndex) { + builder.appendNull(); + } currentEntryBuilder = builder.beginBlockEntry(); } for (BlockConverter converter : converters) { @@ -763,6 +768,10 @@ public void end() for (BlockConverter converter : converters) { converter.afterValue(); } + while (currentEntryBuilder.getPositionCount() < converters.size()) { + currentEntryBuilder.appendNull(); + } + if (builder == null) { nullBuilder.closeEntry(); } @@ -791,13 +800,14 @@ private static class ParquetListConverter private static final int NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD = 32768; private final Type arrayType; + private final int fieldIndex; private final BlockConverter elementConverter; private BlockBuilder builder; private BlockBuilder nullBuilder; // used internally when builder is set to null private BlockBuilder currentEntryBuilder; - public ParquetListConverter(Type prestoType, String columnName, GroupType listType) + public ParquetListConverter(Type prestoType, String columnName, GroupType listType, int fieldIndex) { checkArgument(listType.getFieldCount() == 1, "Expected LIST column '%s' to only have one field, but has %s fields", @@ -806,6 +816,7 @@ public ParquetListConverter(Type prestoType, String columnName, GroupType listTy checkArgument(ARRAY.equals(prestoType.getTypeSignature().getBase())); this.arrayType = prestoType; + this.fieldIndex = fieldIndex; // The Parquet specification requires that the element value of a // LIST type be wrapped in an inner repeated group, like so: @@ -821,7 +832,7 @@ public ParquetListConverter(Type prestoType, String columnName, GroupType listTy // documentation at http://git.io/vOpNz. parquet.schema.Type elementType = listType.getType(0); if (isElementType(elementType, listType.getName())) { - elementConverter = createConverter(prestoType.getTypeParameters().get(0), columnName + ".element", elementType); + elementConverter = createConverter(prestoType.getTypeParameters().get(0), columnName + ".element", elementType, 0); } else { elementConverter = new ParquetListEntryConverter(prestoType.getTypeParameters().get(0), columnName, elementType.asGroupType()); @@ -875,6 +886,9 @@ public void start() currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { + while (builder.getPositionCount() < fieldIndex) { + builder.appendNull(); + } currentEntryBuilder = builder.beginBlockEntry(); } elementConverter.beforeValue(currentEntryBuilder); @@ -926,7 +940,7 @@ public ParquetListEntryConverter(Type prestoType, String columnName, GroupType e columnName, elementType.getFieldCount()); - elementConverter = createConverter(prestoType, columnName + ".element", elementType.getType(0)); + elementConverter = createConverter(prestoType, columnName + ".element", elementType.getType(0), 0); } @Override @@ -969,13 +983,14 @@ private static class ParquetMapConverter private static final int NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD = 32768; private final Type mapType; + private final int fieldIndex; private final ParquetMapEntryConverter entryConverter; private BlockBuilder builder; private BlockBuilder nullBuilder; // used internally when builder is set to null private BlockBuilder currentEntryBuilder; - public ParquetMapConverter(Type type, String columnName, GroupType mapType) + public ParquetMapConverter(Type type, String columnName, GroupType mapType, int fieldIndex) { checkArgument(mapType.getFieldCount() == 1, "Expected MAP column '%s' to only have one field, but has %s fields", @@ -983,6 +998,7 @@ public ParquetMapConverter(Type type, String columnName, GroupType mapType) mapType.getFieldCount()); this.mapType = type; + this.fieldIndex = fieldIndex; parquet.schema.Type entryType = mapType.getFields().get(0); @@ -1014,6 +1030,9 @@ public void start() currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { + while (builder.getPositionCount() < fieldIndex) { + builder.appendNull(); + } currentEntryBuilder = builder.beginBlockEntry(); } entryConverter.beforeValue(currentEntryBuilder); @@ -1084,8 +1103,8 @@ public ParquetMapEntryConverter(Type prestoType, String columnName, GroupType en columnName, entryGroupType.getType(0)); - keyConverter = createConverter(prestoType.getTypeParameters().get(0), columnName + ".key", entryGroupType.getFields().get(0)); - valueConverter = createConverter(prestoType.getTypeParameters().get(1), columnName + ".value", entryGroupType.getFields().get(1)); + keyConverter = createConverter(prestoType.getTypeParameters().get(0), columnName + ".key", entryGroupType.getFields().get(0), 0); + valueConverter = createConverter(prestoType.getTypeParameters().get(1), columnName + ".value", entryGroupType.getFields().get(1), 1); } @Override @@ -1131,12 +1150,14 @@ private static class ParquetPrimitiveConverter implements BlockConverter { private final Type type; + private final int fieldIndex; private BlockBuilder builder; private boolean wroteValue; - public ParquetPrimitiveConverter(Type type) + public ParquetPrimitiveConverter(Type type, int fieldIndex) { this.type = type; + this.fieldIndex = fieldIndex; } @Override @@ -1149,11 +1170,13 @@ public void beforeValue(BlockBuilder builder) @Override public void afterValue() { - if (wroteValue) { - return; - } + } - builder.appendNull(); + private void addMissingValues() + { + while (builder.getPositionCount() < fieldIndex) { + builder.appendNull(); + } } @Override @@ -1187,6 +1210,7 @@ public void addValueFromDictionary(int dictionaryId) @Override public void addBoolean(boolean value) { + addMissingValues(); BOOLEAN.writeBoolean(builder, value); wroteValue = true; } @@ -1194,6 +1218,7 @@ public void addBoolean(boolean value) @Override public void addDouble(double value) { + addMissingValues(); DOUBLE.writeDouble(builder, value); wroteValue = true; } @@ -1201,6 +1226,7 @@ public void addDouble(double value) @Override public void addLong(long value) { + addMissingValues(); BIGINT.writeLong(builder, value); wroteValue = true; } @@ -1208,6 +1234,7 @@ public void addLong(long value) @Override public void addBinary(Binary value) { + addMissingValues(); if (type == TIMESTAMP) { builder.writeLong(ParquetTimestampUtils.getTimestampMillis(value)).closeEntry(); } @@ -1220,6 +1247,7 @@ public void addBinary(Binary value) @Override public void addFloat(float value) { + addMissingValues(); DOUBLE.writeDouble(builder, value); wroteValue = true; } @@ -1227,6 +1255,7 @@ public void addFloat(float value) @Override public void addInt(int value) { + addMissingValues(); BIGINT.writeLong(builder, value); wroteValue = true; } From 3bbcbb77181ef0d9a543dcbcceb988fcfad0de39 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Thu, 1 Oct 2015 11:27:09 -0700 Subject: [PATCH 03/83] [maven-release-plugin] prepare for tw-0.1 release. --- pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 35 files changed, 35 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index 1ce3b8933afa..2e906d46ab60 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT pom presto-root diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index bf0ea89449f7..becc2aad6552 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index bea67680cd7e..fcc11ed9be7d 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index eea12d2ea187..3dca897d8d00 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 4bd015fe78bf..5f68c004c6e3 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index e4f4e3b4f115..b4276f9a2118 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 812de79ce39b..e0fa76890e65 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 53672dfabdb5..33caecd03adb 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 460610625774..22a19de7cdf6 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 8b44291c6a43..c805d506939e 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index cac75be155a5..4c764cc832c3 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 3caded1f3f89..f775951da3c0 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 01deea780ac4..28bba6912311 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 9028adcbcaf7..0b47093c5d7a 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index a0edca6dd537..40bfd35c4cd0 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9f6939498365..198b7dba9b74 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 4dbffd33d13f..11364ccb4163 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index bf3b0a07e618..727e8efad01c 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index a8b129bfcf72..f2a2d446e7d3 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 4f2b30229f50..d4678be9d7e6 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 604afe0f5e1f..3005cbee88c0 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 6fe0e61a7972..2d9f2539d18a 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index eef17295c589..f99df2a20914 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 755c12ab097f..2e3f97bf0bf4 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-postgresql diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 83d83e4accff..72ce96885268 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 25555b4a25d2..6d344eada7e4 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 8d44339c6c78..1dc2d16c7fb8 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 33e66de413fc..30e1a7c0a1e8 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index ac51dbc823c4..468278f4a955 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index ad8a095f4f39..2e9c6a0c9c13 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 40bafdb094d3..05fe47646434 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index a321b43f5771..d30e0605f888 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 44501fcdf2bf..9ec7a73cd2da 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index ea19dd90dcf6..932924f8b8ed 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 3491c458ba4d..8cade2a0ed97 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-verifier From ca8560fde9575f6d45dd6b2d24c5775105c5bf32 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Thu, 1 Oct 2015 11:42:10 -0700 Subject: [PATCH 04/83] [maven-release-plugin] prepare for release 0.119-tw-0.1. --- pom.xml | 8 ++++---- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 35 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 2e906d46ab60..73f572f41e55 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 pom presto-root @@ -28,9 +28,9 @@ - scm:git:git://github.com/facebook/presto.git - https://github.com/facebook/presto - HEAD + scm:git:git://github.com/twitter-forks/presto.git + https://github.com/twitter-forks/presto + 0.119-tw-0.1 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index becc2aad6552..bf85b92c69ab 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index fcc11ed9be7d..dabd15b59a88 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 3dca897d8d00..7900d3a15d20 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 5f68c004c6e3..1d62047b7b8c 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index b4276f9a2118..85b7806119be 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index e0fa76890e65..53f55d5d5c91 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 33caecd03adb..242560f261a8 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 22a19de7cdf6..364b9e3e3825 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index c805d506939e..07d4930b6b8b 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 4c764cc832c3..1af8e55a0d46 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index f775951da3c0..fa858095ac23 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 28bba6912311..99226c90a798 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 0b47093c5d7a..b0a353cd0575 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 40bfd35c4cd0..b94aaa9676fd 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 198b7dba9b74..f4a2653f06a2 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 11364ccb4163..5c1c1418f45c 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 727e8efad01c..029add19febd 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index f2a2d446e7d3..5bdea99bb96a 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index d4678be9d7e6..911c908772bb 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 3005cbee88c0..21d1697a581b 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 2d9f2539d18a..5730006d2752 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index f99df2a20914..f14de7f6b725 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 2e3f97bf0bf4..a43d0b35f4a0 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-postgresql diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 72ce96885268..9a8940684940 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 6d344eada7e4..506e6e6fb4d8 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 1dc2d16c7fb8..da5609eb349d 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 30e1a7c0a1e8..2c9e1c9a8c72 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 468278f4a955..9c258c8b88dc 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 2e9c6a0c9c13..b41a6498d846 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 05fe47646434..1ee4d99cfda8 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index d30e0605f888..661f3db9a240 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 9ec7a73cd2da..f76a75aaba9b 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 932924f8b8ed..6b3161861e6f 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 8cade2a0ed97..c22f6a70eb88 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-verifier From ec109d1dce3d3d93a9335f3750c8bd3c424bb22e Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 2 Oct 2015 15:47:29 -0700 Subject: [PATCH 05/83] Compare types ignoring cases. This is not complete solution and should be reverted once #2863 is resolved. --- .../java/com/facebook/presto/metadata/FunctionRegistry.java | 2 +- .../com/facebook/presto/operator/scalar/RowFieldReference.java | 3 +-- .../com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java | 2 +- .../main/java/com/facebook/presto/type/RowParametricType.java | 2 +- .../main/java/com/facebook/presto/spi/type/TypeSignature.java | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java index 8c4d9a48af1d..45bff811f1a1 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java @@ -449,7 +449,7 @@ public FunctionInfo resolveFunction(QualifiedName name, List para RowType rowType = RowParametricType.ROW.createType(resolveTypes(typeSignature.getParameters(), typeManager), typeSignature.getLiteralParameters()); // search for exact match for (ParametricFunction function : RowParametricType.ROW.createFunctions(rowType)) { - if (!function.getSignature().getName().equals(name.toString())) { + if (!function.getSignature().getName().equalsIgnoreCase(name.toString())) { continue; } Map boundTypeParameters = function.getSignature().bindTypeParameters(resolvedTypes, false, typeManager); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowFieldReference.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowFieldReference.java index f4a94125f49a..4291e7970fc9 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowFieldReference.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowFieldReference.java @@ -28,7 +28,6 @@ import java.lang.invoke.MethodHandle; import java.util.Map; -import java.util.Optional; import static com.facebook.presto.sql.QueryUtil.mangleFieldReference; import static com.facebook.presto.type.RowType.RowField; @@ -59,7 +58,7 @@ public RowFieldReference(RowType type, String fieldName) Type returnType = null; int index = 0; for (RowField field : type.getFields()) { - if (field.getName().equals(Optional.of(fieldName))) { + if (field.getName().get().equalsIgnoreCase(fieldName)) { returnType = field.getType(); break; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java index 22b3d3559ec0..6239c559d708 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java @@ -318,7 +318,7 @@ private Type tryVisitRowFieldAccessor(QualifiedNameReference node) RowType rowType = checkType(field.getType(), RowType.class, "field.getType()"); Type rowFieldType = null; for (RowField rowField : rowType.getFields()) { - if (rowField.getName().equals(Optional.of(node.getName().getSuffix()))) { + if (rowField.getName().get().equalsIgnoreCase(node.getName().getSuffix())) { rowFieldType = rowField.getType(); break; } diff --git a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java index 5a4804d99868..f175754d696a 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java +++ b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java @@ -65,7 +65,7 @@ public List createFunctions(Type type) ImmutableList.Builder builder = ImmutableList.builder(); for (RowField field : rowType.getFields()) { field.getName() - .ifPresent(name -> builder.add(new RowFieldReference(rowType, field.getName().get()))); + .ifPresent(name -> builder.add(new RowFieldReference(rowType, field.getName().get().toLowerCase()))); } return builder.build(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java index 8e5d924fe399..75ffe4906f81 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java @@ -189,7 +189,7 @@ private static Object parseLiteral(String literal) { if (literal.startsWith("'") || literal.endsWith("'")) { checkArgument(literal.startsWith("'") && literal.endsWith("'"), "Bad literal: '%s'", literal); - return literal.substring(1, literal.length() - 1); + return literal.substring(1, literal.length() - 1).toLowerCase(); } else { return Long.parseLong(literal); From c76b42e910bfbf989078171587dc6700f2757275 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 9 Oct 2015 12:43:09 -0700 Subject: [PATCH 06/83] Revert "Reduce allocation of hadoop configuration objects" This reverts commit 7365f1ad3a75b478bcdecca65676d7857b843699. This blocked us from reading from mountable clusters. --- .../com/facebook/presto/hive/HiveHdfsConfiguration.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java index 8ad5acc65c9f..dc6d0f46f3ed 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java @@ -18,25 +18,19 @@ import javax.inject.Inject; import java.net.URI; -import java.util.Map; import static java.util.Objects.requireNonNull; public class HiveHdfsConfiguration implements HdfsConfiguration { - private static final Configuration DEFAULT_CONFIGURATION = new Configuration(); - @SuppressWarnings("ThreadLocalNotStaticFinal") private final ThreadLocal hadoopConfiguration = new ThreadLocal() { @Override protected Configuration initialValue() { - Configuration config = new Configuration(false); - for (Map.Entry entry : DEFAULT_CONFIGURATION) { - config.set(entry.getKey(), entry.getValue()); - } + Configuration config = new Configuration(); updater.updateConfiguration(config); return config; } From f0d9ab0d9780de736d7c6893ccf4960a57dc8fba Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Tue, 13 Oct 2015 14:25:46 -0700 Subject: [PATCH 07/83] Bump up zookeeper version to 3.4.6 --- pom.xml | 10 +++++++++- .../facebook/presto/kafka/util/EmbeddedZookeeper.java | 7 ++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 301487adddb2..c845b238ec5b 100644 --- a/pom.xml +++ b/pom.xml @@ -624,7 +624,7 @@ org.apache.zookeeper zookeeper - 3.3.6 + 3.4.6 junit @@ -634,6 +634,14 @@ log4j log4j + + org.slf4j + slf4j-jdk14 + + + org.slf4j + slf4j-log4j12 + diff --git a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java index fddf5c845746..0f319cee1cbc 100644 --- a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java +++ b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java @@ -14,7 +14,7 @@ package com.facebook.presto.kafka.util; import com.google.common.io.Files; -import org.apache.zookeeper.server.NIOServerCnxn; +import org.apache.zookeeper.server.NIOServerCnxnFactory; import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; @@ -32,7 +32,7 @@ public class EmbeddedZookeeper private final int port; private final File zkDataDir; private final ZooKeeperServer zkServer; - private final NIOServerCnxn.Factory cnxnFactory; + private final NIOServerCnxnFactory cnxnFactory; private final AtomicBoolean started = new AtomicBoolean(); private final AtomicBoolean stopped = new AtomicBoolean(); @@ -53,7 +53,8 @@ public EmbeddedZookeeper(int port) FileTxnSnapLog ftxn = new FileTxnSnapLog(zkDataDir, zkDataDir); zkServer.setTxnLogFactory(ftxn); - cnxnFactory = new NIOServerCnxn.Factory(new InetSocketAddress(this.port), 0); + cnxnFactory = new NIOServerCnxnFactory(); + cnxnFactory.configure(new InetSocketAddress(this.port), 0); } public void start() From 3bbabcefe030d9794fd329a4ec927f19cb75faca Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Tue, 13 Oct 2015 14:47:57 -0700 Subject: [PATCH 08/83] Read metastore info from zookeeper. --- presto-hive/pom.xml | 24 ++++ .../presto/hive/HiveClientModule.java | 4 +- .../hive/ZookeeperMetastoreMonitor.java | 125 ++++++++++++++++++ .../hive/ZookeeperServersetHiveCluster.java | 67 ++++++++++ .../ZookeeperServersetMetastoreConfig.java | 87 ++++++++++++ ...TestZookeeperServersetMetastoreConfig.java | 55 ++++++++ 6 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 94423d7afb8b..a8f459673f39 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -43,6 +43,24 @@ hive-apache + + org.apache.curator + curator-recipes + 2.8.0 + + + + org.apache.curator + curator-framework + 2.8.0 + + + + org.apache.curator + curator-client + 2.8.0 + + org.apache.thrift libthrift @@ -88,6 +106,12 @@ configuration + + com.googlecode.json-simple + json-simple + 1.1 + + com.google.guava guava diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index 6a2e659ae27c..cc540c3076ea 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -90,8 +90,8 @@ public void configure(Binder binder) newExporter(binder).export(NamenodeStats.class).as(generatedNameOf(NamenodeStats.class)); binder.bind(HiveMetastoreClientFactory.class).in(Scopes.SINGLETON); - binder.bind(HiveCluster.class).to(StaticHiveCluster.class).in(Scopes.SINGLETON); - configBinder(binder).bindConfig(StaticMetastoreConfig.class); + binder.bind(HiveCluster.class).to(ZookeeperServersetHiveCluster.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(ZookeeperServersetMetastoreConfig.class); binder.bind(TypeManager.class).toInstance(typeManager); binder.bind(PageIndexerFactory.class).toInstance(pageIndexerFactory); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java new file mode 100644 index 000000000000..002eefe5cc13 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java @@ -0,0 +1,125 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.common.net.HostAndPort; + +import io.airlift.log.Logger; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.utils.ZKPaths; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +public class ZookeeperMetastoreMonitor implements PathChildrenCacheListener +{ + public static final Logger log = Logger.get(ZookeeperMetastoreMonitor.class); + private CuratorFramework client; + private PathChildrenCache cache; + private ConcurrentMap servers; // (Node_Name->HostAndPort) + + public ZookeeperMetastoreMonitor(String zkServer, String watchPath, int maxRetries, int retrySleepTime) + throws Exception + { + client = CuratorFrameworkFactory.newClient(zkServer, new ExponentialBackoffRetry(retrySleepTime, maxRetries)); + client.start(); + + cache = new PathChildrenCache(client, watchPath, true); // true indicating cache node contents in addition to the stat + try { + cache.start(); + } + catch (Exception ex) { + throw new RuntimeException("Curator PathCache Creation failed: " + ex.getMessage()); + } + + cache.getListenable().addListener(this); + servers = new ConcurrentHashMap<>(); + } + + public void close() + { + client.close(); + + try { + cache.close(); + } + catch (IOException ex) { + // do nothing + } + } + + public List getServers() + { + return servers.values().stream().collect(Collectors.toList()); + } + + private HostAndPort deserialize(byte[] bytes) + { + String serviceEndpoint = "serviceEndpoint"; + JSONObject data = (JSONObject) JSONValue.parse(new String(bytes)); + if (data != null && data.containsKey(serviceEndpoint)) { + Map hostPortMap = (Map) data.get(serviceEndpoint); + String host = hostPortMap.get("host").toString(); + int port = Integer.parseInt(hostPortMap.get("port").toString()); + return HostAndPort.fromParts(host, port); + } + else { + log.warn("failed to deserialize child node data"); + throw new IllegalArgumentException("No host:port found"); + } + } + + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception + { + String node = ZKPaths.getNodeFromPath(event.getData().getPath()); + + switch (event.getType()) { + case CHILD_ADDED: { + HostAndPort hostPort = deserialize(event.getData().getData()); + log.info("child added: " + node + ": " + hostPort); + servers.put(node, hostPort); + break; + } + + case CHILD_UPDATED: { + HostAndPort hostPort = deserialize(event.getData().getData()); + log.info("child updated: " + node + ": " + hostPort); + servers.put(node, hostPort); + break; + } + + case CHILD_REMOVED: { + log.info("child removed: " + node); + servers.remove(node); + break; + } + + default: + log.info("connection state changed: " + node); + break; + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java new file mode 100644 index 000000000000..57b44d5bc05e --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.spi.PrestoException; +import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; +import org.apache.thrift.transport.TTransportException; + +import javax.inject.Inject; + +import java.util.List; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; +import static java.util.Objects.requireNonNull; + +/** + * Created by smittal on 10/9/15. + */ +public class ZookeeperServersetHiveCluster + implements HiveCluster +{ + private static final Logger log = Logger.get(ZookeeperServersetHiveCluster.class); + private final HiveMetastoreClientFactory clientFactory; + private ZookeeperMetastoreMonitor zkMetastoreMonitor; + + @Inject + public ZookeeperServersetHiveCluster(ZookeeperServersetMetastoreConfig config, HiveMetastoreClientFactory clientFactory) + throws Exception + { + String zkServerHostAndPort = requireNonNull(config.getZookeeperServerHostAndPort(), "zkServerHostAndPort is null"); + String zkMetastorePath = requireNonNull(config.getZookeeperMetastorePath(), "zkMetastorePath is null"); + int zkRetries = requireNonNull(config.getZookeeperMaxRetries(), "zkMaxRetried is null"); + int zkRetrySleepTime = requireNonNull(config.getZookeeperRetrySleepTime(), "zkRetrySleepTime is null"); + this.clientFactory = requireNonNull(clientFactory, "clientFactory is null"); + this.zkMetastoreMonitor = new ZookeeperMetastoreMonitor(zkServerHostAndPort, zkMetastorePath, zkRetries, zkRetrySleepTime); + } + + @Override + public HiveMetastoreClient createMetastoreClient() + { + List metastores = zkMetastoreMonitor.getServers(); + TTransportException lastException = null; + for (HostAndPort metastore : metastores) { + try { + log.info("Connecting to metastore at: " + metastore.toString()); + return clientFactory.create(metastore.getHostText(), metastore.getPort()); + } + catch (TTransportException e) { + lastException = e; + } + } + + throw new PrestoException(HIVE_METASTORE_ERROR, "Failed connecting to Hive metastore", lastException); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java new file mode 100644 index 000000000000..339131fa3b9b --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + * Created by smittal on 10/9/15. + */ +public class ZookeeperServersetMetastoreConfig +{ + private String zookeeperServerHostAndPort; + private String zookeeperMetastorePath; + private int zookeeperRetrySleepTime = 500; // ms + private int zookeeperMaxRetries = 3; + + @NotNull + public String getZookeeperServerHostAndPort() + { + return zookeeperServerHostAndPort; + } + + @Config("hive.metastore.zookeeper.uri") + @ConfigDescription("Zookeeper Host and Port") + public ZookeeperServersetMetastoreConfig setZookeeperServerHostAndPort(String zookeeperServerHostAndPort) + { + this.zookeeperServerHostAndPort = zookeeperServerHostAndPort; + return this; + } + + @NotNull + public String getZookeeperMetastorePath() + { + return zookeeperMetastorePath; + } + + @Config("hive.metastore.zookeeper.path") + @ConfigDescription("Hive metastore Zookeeper path") + public ZookeeperServersetMetastoreConfig setZookeeperMetastorePath(String zkPath) + { + this.zookeeperMetastorePath = zkPath; + return this; + } + + @NotNull + public int getZookeeperRetrySleepTime() + { + return zookeeperRetrySleepTime; + } + + @Config("hive.metastore.zookeeper.retry.sleeptime") + @ConfigDescription("Zookeeper sleep time between reties") + public ZookeeperServersetMetastoreConfig setZookeeperRetrySleepTime(int zookeeperRetrySleepTime) + { + this.zookeeperRetrySleepTime = zookeeperRetrySleepTime; + return this; + } + + @Min(1) + public int getZookeeperMaxRetries() + { + return zookeeperMaxRetries; + } + + @Config("hive.metastore.zookeeper.max.retries") + @ConfigDescription("Zookeeper max reties") + public ZookeeperServersetMetastoreConfig setZookeeperMaxRetries(int zookeeperMaxRetries) + { + this.zookeeperMaxRetries = zookeeperMaxRetries; + return this; + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java new file mode 100644 index 000000000000..bb1117697296 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestZookeeperServersetMetastoreConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(ZookeeperServersetMetastoreConfig.class) + .setZookeeperMaxRetries(3) + .setZookeeperRetrySleepTime(500) + .setZookeeperMetastorePath(null) + .setZookeeperServerHostAndPort(null)); + } + + @Test + public void testExplicitPropertyMappingsSingleMetastore() + { + Map properties = new ImmutableMap.Builder() + .put("hive.metastore.zookeeper.uri", "localhost:2181") + .put("hive.metastore.zookeeper.path", "/zookeeper/path/") + .put("hive.metastore.zookeeper.retry.sleeptime", "200") + .put("hive.metastore.zookeeper.max.retries", "2") + .build(); + + ZookeeperServersetMetastoreConfig expected = new ZookeeperServersetMetastoreConfig() + .setZookeeperServerHostAndPort("localhost:2181") + .setZookeeperMetastorePath("/zookeeper/path/") + .setZookeeperRetrySleepTime(200) + .setZookeeperMaxRetries(2); + + assertFullMapping(properties, expected); + } +} From 202e620ec7f7743d0d0910573eef607c2eb2cd1f Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Tue, 13 Oct 2015 13:52:37 -0700 Subject: [PATCH 09/83] Add some tests and fix injection issue. --- presto-hive/pom.xml | 20 +++ .../presto/hive/HiveClientModule.java | 2 - .../presto/hive/HiveConnectorFactory.java | 8 + .../hive/MetastoreStaticClusterModule.java | 31 ++++ .../hive/MetastoreZkDiscoveryBasedModule.java | 31 ++++ .../ZookeeperServersetMetastoreConfig.java | 2 - .../hive/TestZookeeperMetastoreMonitor.java | 157 ++++++++++++++++++ .../facebook/presto/hive/util/TestUtils.java | 30 ++++ 8 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index a8f459673f39..e75baa539500 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -61,6 +61,26 @@ 2.8.0 + + org.apache.curator + curator-test + 2.8.0 + test + + + + org.apache.zookeeper + zookeeper + 3.4.6 + test + + + + com.101tec + zkclient + test + + org.apache.thrift libthrift diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index cc540c3076ea..bc48cbfc7f0f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -90,8 +90,6 @@ public void configure(Binder binder) newExporter(binder).export(NamenodeStats.class).as(generatedNameOf(NamenodeStats.class)); binder.bind(HiveMetastoreClientFactory.class).in(Scopes.SINGLETON); - binder.bind(HiveCluster.class).to(ZookeeperServersetHiveCluster.class).in(Scopes.SINGLETON); - configBinder(binder).bindConfig(ZookeeperServersetMetastoreConfig.class); binder.bind(TypeManager.class).toInstance(typeManager); binder.bind(PageIndexerFactory.class).toInstance(pageIndexerFactory); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java index 2725e938586e..922875f49317 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java @@ -93,6 +93,14 @@ public Connector create(String connectorId, Map config) new MBeanModule(), new JsonModule(), new HiveClientModule(connectorId, metastore, typeManager, pageIndexerFactory), + installModuleIf( + ZookeeperServersetMetastoreConfig.class, + zkMetastoreConfig -> zkMetastoreConfig.getZookeeperServerHostAndPort() == null, + new MetastoreStaticClusterModule()), + installModuleIf( + ZookeeperServersetMetastoreConfig.class, + zkMetastoreConfig -> zkMetastoreConfig.getZookeeperServerHostAndPort() != null, + new MetastoreZkDiscoveryBasedModule()), installModuleIf( SecurityConfig.class, security -> "none".equalsIgnoreCase(security.getSecuritySystem()), diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java new file mode 100644 index 000000000000..18bfa3d1eff1 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static io.airlift.configuration.ConfigBinder.configBinder; + +public class MetastoreStaticClusterModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(HiveCluster.class).to(StaticHiveCluster.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(StaticMetastoreConfig.class); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java new file mode 100644 index 000000000000..a4d84813ab76 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static io.airlift.configuration.ConfigBinder.configBinder; + +public class MetastoreZkDiscoveryBasedModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(HiveCluster.class).to(ZookeeperServersetHiveCluster.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(ZookeeperServersetMetastoreConfig.class); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java index 339131fa3b9b..60867659d906 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java @@ -29,7 +29,6 @@ public class ZookeeperServersetMetastoreConfig private int zookeeperRetrySleepTime = 500; // ms private int zookeeperMaxRetries = 3; - @NotNull public String getZookeeperServerHostAndPort() { return zookeeperServerHostAndPort; @@ -43,7 +42,6 @@ public ZookeeperServersetMetastoreConfig setZookeeperServerHostAndPort(String zo return this; } - @NotNull public String getZookeeperMetastorePath() { return zookeeperMetastorePath; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java new file mode 100644 index 000000000000..5383e2e64198 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.hadoop.shaded.com.google.common.collect.ImmutableList; +import com.facebook.presto.hive.util.TestUtils; +import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; +import org.I0Itec.zkclient.ZkClient; +import org.I0Itec.zkclient.exception.ZkMarshallingError; +import org.I0Itec.zkclient.serialize.ZkSerializer; +import org.apache.curator.test.TestingServer; +import org.json.simple.JSONObject; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.testng.Assert.assertTrue; + +public class TestZookeeperMetastoreMonitor +{ + private static final Logger log = Logger.get(TestZookeeperMetastoreMonitor.class); + + private ZookeeperMetastoreMonitor zkMetastoreMonitor; + private TestingServer zkServer; + private ZkClient zkClient; + private final String zkPath = "/metastores"; + + public TestZookeeperMetastoreMonitor() + throws Exception + { + zkServer = new TestingServer(TestUtils.findUnusedPort()); + zkClient = new ZkClient(zkServer.getConnectString(), 30_000, 30_000); + + // Set the serializer + zkClient.setZkSerializer(new ZkSerializer() { + @Override + public byte[] serialize(Object o) throws ZkMarshallingError + { + try { + return o.toString().getBytes("UTF-8"); + } + catch (Exception e) { + log.warn("Exception in serializing " + e); + } + return "".getBytes(); + } + + @Override + public Object deserialize(byte[] bytes) throws ZkMarshallingError + { + return null; + } + }); + } + + @AfterClass + public void destroy() + throws IOException + { + zkMetastoreMonitor.close(); + zkClient.close(); + zkServer.close(); + + } + + @BeforeTest + public void setUp() + throws Exception + { + log.info("Cleaning up zookeeper"); + zkClient.getChildren("/").stream() + .filter(child -> !child.equals("zookeeper")) + .forEach(child -> zkClient.deleteRecursive("/" + child)); + + zkClient.unsubscribeAll(); + + zkClient.createPersistent(zkPath); + zkMetastoreMonitor = new ZookeeperMetastoreMonitor(zkServer.getConnectString(), zkPath, 3, 500); + } + + @Test + public void testGetServers() throws Exception + { + List servers; + List expected; + assertTrue(zkMetastoreMonitor.getServers().isEmpty()); + + addServerToZk("nameNode1", "host1", 10001); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host1", 10001)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers)); + + addServerToZk("nameNode2", "host2", 10002); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host1", 10001), HostAndPort.fromParts("host2", 10002)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers)); + + // Change value of an existing name node + addServerToZk("nameNode2", "host2", 10003); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host1", 10001), HostAndPort.fromParts("host2", 10003)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers)); + + // Delete an existing name node + zkClient.delete(getPathForNameNode("nameNode1")); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host2", 10003)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers), servers.toString()); + } + + private void addServerToZk(String nameNode, String host, int port) + { + JSONObject serviceEndpoint = new JSONObject(); + serviceEndpoint.put("host", host); + serviceEndpoint.put("port", port); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("serviceEndpoint", serviceEndpoint); + + String path = getPathForNameNode(nameNode); + + if (!zkClient.exists(path)) { + zkClient.createPersistent(path, jsonObject.toJSONString()); + } + else { + zkClient.writeData(path, jsonObject.toJSONString()); + } + } + + private String getPathForNameNode(String nameNode) + { + return zkPath + "/" + nameNode; + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java new file mode 100644 index 000000000000..4315bac667c2 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.util; + +import java.io.IOException; +import java.net.ServerSocket; + +public final class TestUtils +{ + private TestUtils() {} + + public static int findUnusedPort() + throws IOException + { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } +} From 93f55bbf1ebda92e6f4bf4929ba36ef8dadcd292 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Wed, 14 Oct 2015 09:59:22 -0700 Subject: [PATCH 10/83] Remove Created by comments added by Idea. --- .../facebook/presto/hive/ZookeeperServersetHiveCluster.java | 3 --- .../presto/hive/ZookeeperServersetMetastoreConfig.java | 3 --- 2 files changed, 6 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java index 57b44d5bc05e..cbc434b74d2f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java @@ -25,9 +25,6 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static java.util.Objects.requireNonNull; -/** - * Created by smittal on 10/9/15. - */ public class ZookeeperServersetHiveCluster implements HiveCluster { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java index 60867659d906..44c6f9d19188 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java @@ -19,9 +19,6 @@ import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; -/** - * Created by smittal on 10/9/15. - */ public class ZookeeperServersetMetastoreConfig { private String zookeeperServerHostAndPort; From 608333956fd263a0e3e103b1c52221c4d33f45fd Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Thu, 22 Oct 2015 13:42:25 -0700 Subject: [PATCH 11/83] Fix some warnings after merging latest master. --- .../facebook/presto/hive/ZookeeperServersetHiveCluster.java | 1 + .../facebook/presto/hive/TestZookeeperMetastoreMonitor.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java index cbc434b74d2f..48f869ec8ed0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.HiveMetastoreClient; import com.facebook.presto.spi.PrestoException; import com.google.common.net.HostAndPort; import io.airlift.log.Logger; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java index 5383e2e64198..9ccf999edac4 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java @@ -27,6 +27,7 @@ import org.testng.annotations.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.TimeUnit; @@ -53,7 +54,7 @@ public TestZookeeperMetastoreMonitor() public byte[] serialize(Object o) throws ZkMarshallingError { try { - return o.toString().getBytes("UTF-8"); + return o.toString().getBytes(StandardCharsets.UTF_8); } catch (Exception e) { log.warn("Exception in serializing " + e); From 972d5d166bc8174c9cef50b1066f6efc9a07a02b Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 23 Oct 2015 11:39:02 -0700 Subject: [PATCH 12/83] Create ParquetHiveRecordCursor as user, so that hdfs reads happen as that user. --- .../parquet/ParquetRecordCursorProvider.java | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java index eb6bb880bed3..3854449498da 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java @@ -27,10 +27,12 @@ import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; import org.joda.time.DateTimeZone; import javax.inject.Inject; +import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -88,20 +90,26 @@ public Optional createHiveRecordCursor( throw new IllegalArgumentException("Can not read Parquet column: " + unsupportedColumns); } - return Optional.of(new ParquetHiveRecordCursor( - configuration, - path, - start, - length, - schema, - partitionKeys, - columns, - useParquetColumnNames, - hiveStorageTimeZone, - typeManager, - isParquetPredicatePushdownEnabled(session), - effectivePredicate - )); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); + try { + return ugi.doAs((PrivilegedExceptionAction>) () -> Optional.of(new ParquetHiveRecordCursor( + configuration, + path, + start, + length, + schema, + partitionKeys, + columns, + useParquetColumnNames, + hiveStorageTimeZone, + typeManager, + isParquetPredicatePushdownEnabled(session), + effectivePredicate + ))); + } + catch (Exception e) { + throw new RuntimeException(e); + } } private static Predicate isParquetSupportedType() From d01400182645e82f733760a2d938273c49dd5a88 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 13:20:43 -0700 Subject: [PATCH 13/83] Propagate user session information throughout hive metastore. --- .../facebook/presto/hive/HiveMetadata.java | 85 ++++++++++--------- .../facebook/presto/hive/HivePageSink.java | 11 ++- .../presto/hive/HivePageSinkProvider.java | 7 +- .../presto/hive/HivePartitionManager.java | 12 +-- .../presto/hive/HiveSplitManager.java | 7 +- .../facebook/presto/hive/HiveWriteUtils.java | 9 +- .../hive/metastore/CachingHiveMetastore.java | 30 +++---- .../presto/hive/metastore/HiveMetastore.java | 34 ++++---- .../presto/hive/AbstractTestHiveClient.java | 8 +- .../presto/hive/AbstractTestHiveClientS3.java | 20 ++--- .../hive/metastore/InMemoryHiveMetastore.java | 32 +++---- .../metastore/TestCachingHiveMetastore.java | 59 ++++++------- 12 files changed, 164 insertions(+), 150 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index c8e48aa7e21b..a3ad7180b9de 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -231,14 +231,14 @@ public HivePartitionManager getPartitionManager() @Override public List listSchemaNames(ConnectorSession session) { - return metastore.getAllDatabases(); + return metastore.getAllDatabases(session.getUser()); } @Override public HiveTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { requireNonNull(tableName, "tableName is null"); - if (!metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).isPresent()) { + if (!metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()).isPresent()) { return null; } return new HiveTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName()); @@ -249,12 +249,12 @@ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, Connect { requireNonNull(tableHandle, "tableHandle is null"); SchemaTableName tableName = schemaTableName(tableHandle); - return getTableMetadata(tableName); + return getTableMetadata(session, tableName); } - private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) + private ConnectorTableMetadata getTableMetadata(ConnectorSession session, SchemaTableName tableName) { - Optional table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) { throw new TableNotFoundException(tableName); } @@ -294,7 +294,7 @@ public List listTables(ConnectorSession session, String schemaN { ImmutableList.Builder tableNames = ImmutableList.builder(); for (String schemaName : listSchemas(session, schemaNameOrNull)) { - for (String tableName : metastore.getAllTables(schemaName).orElse(emptyList())) { + for (String tableName : metastore.getAllTables(session.getUser(), schemaName).orElse(emptyList())) { tableNames.add(new SchemaTableName(schemaName, tableName)); } } @@ -313,7 +313,7 @@ private List listSchemas(ConnectorSession session, String schemaNameOrNu public ColumnHandle getSampleWeightColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -335,7 +335,7 @@ public boolean canCreateSampledTables(ConnectorSession session) public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -356,7 +356,7 @@ public Map> listTableColumns(ConnectorSess ImmutableMap.Builder> columns = ImmutableMap.builder(); for (SchemaTableName tableName : listTables(session, prefix)) { try { - columns.put(tableName, getTableMetadata(tableName).getColumns()); + columns.put(tableName, getTableMetadata(session, tableName).getColumns()); } catch (HiveViewNotSupportedException e) { // view is not supported @@ -397,17 +397,18 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe List partitionedBy = getPartitionedBy(tableMetadata.getProperties()); List columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); - createTable(schemaName, tableName, tableMetadata.getOwner(), columnHandles, hiveStorageFormat, partitionedBy); + createTable(session, schemaName, tableName, tableMetadata.getOwner(), columnHandles, hiveStorageFormat, partitionedBy); } - public void createTable(String schemaName, + public void createTable(ConnectorSession session, + String schemaName, String tableName, String tableOwner, List columnHandles, HiveStorageFormat hiveStorageFormat, List partitionedBy) { - Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName); + Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName); // verify the target directory for the table if (pathExists(hdfsEnvironment, targetPath)) { @@ -415,10 +416,11 @@ public void createTable(String schemaName, } createDirectory(hdfsEnvironment, targetPath); - createTable(schemaName, tableName, tableOwner, columnHandles, hiveStorageFormat, partitionedBy, targetPath); + createTable(session, schemaName, tableName, tableOwner, columnHandles, hiveStorageFormat, partitionedBy, targetPath); } - private Table createTable(String schemaName, + private Table createTable(ConnectorSession session, + String schemaName, String tableName, String tableOwner, List columnHandles, @@ -484,7 +486,7 @@ else if (!partitionColumnNames.contains(name)) { ImmutableMap.of(), ImmutableMap.of())); - metastore.createTable(table); + metastore.createTable(session.getUser(), table); return table; } @@ -496,7 +498,7 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle } HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); - Optional
tableMetadata = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
tableMetadata = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(handle.getSchemaTableName()); } @@ -509,7 +511,7 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle sd.setCols(columns.build()); table.setSd(sd); - metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); + metastore.alterTable(session.getUser(), handle.getSchemaName(), handle.getTableName(), table); } @Override @@ -521,7 +523,7 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHan HiveTableHandle hiveTableHandle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); HiveColumnHandle sourceHandle = checkType(source, HiveColumnHandle.class, "columnHandle"); - Optional
tableMetadata = metastore.getTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); + Optional
tableMetadata = metastore.getTable(session.getUser(), hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(hiveTableHandle.getSchemaTableName()); } @@ -538,7 +540,7 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHan } sd.setCols(columns.build()); table.setSd(sd); - metastore.alterTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); + metastore.alterTable(session.getUser(), hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); } @Override @@ -550,14 +552,14 @@ public void renameTable(ConnectorSession session, ConnectorTableHandle tableHand HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); SchemaTableName tableName = schemaTableName(tableHandle); - Optional
source = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
source = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!source.isPresent()) { throw new TableNotFoundException(tableName); } Table table = source.get(); table.setDbName(newTableName.getSchemaName()); table.setTableName(newTableName.getTableName()); - metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); + metastore.alterTable(session.getUser(), handle.getSchemaName(), handle.getTableName(), table); } @Override @@ -570,7 +572,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle throw new PrestoException(PERMISSION_DENIED, "DROP TABLE is disabled in this Hive catalog"); } - Optional
target = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
target = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } @@ -579,7 +581,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle if (!session.getUser().equals(table.getOwner())) { throw new PrestoException(PERMISSION_DENIED, format("Unable to drop table '%s': owner of the table is different from session user", table)); } - metastore.dropTable(handle.getSchemaName(), handle.getTableName()); + metastore.dropTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); } @Override @@ -599,7 +601,7 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto List columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); - Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName); + Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName); // verify the target directory for the table if (pathExists(hdfsEnvironment, targetPath)) { @@ -633,7 +635,7 @@ public void commitCreateTable(ConnectorSession session, ConnectorOutputTableHand .map(partitionUpdateCodec::fromJson) .collect(toList()); - Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, handle.getSchemaName(), handle.getTableName()); + Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, handle.getSchemaName(), handle.getTableName()); Path writePath = new Path(handle.getWritePath().get()); // rename if using a temporary directory @@ -649,11 +651,12 @@ public void commitCreateTable(ConnectorSession session, ConnectorOutputTableHand renameDirectory(hdfsEnvironment, handle.getSchemaName(), handle.getTableName(), writePath, targetPath); } - PartitionCommitter partitionCommitter = new PartitionCommitter(handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); + PartitionCommitter partitionCommitter = new PartitionCommitter(session, handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); try { partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates); Table table = createTable( + session, handle.getSchemaName(), handle.getTableName(), handle.getTableOwner(), @@ -691,7 +694,7 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl verifyJvmTimeZone(); SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -724,6 +727,7 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl private static class PartitionCommitter implements Closeable { + private final ConnectorSession session; private final String schemaName; private final String tableName; private final HiveMetastore metastore; @@ -731,8 +735,9 @@ private static class PartitionCommitter private final List batch; private final List createdPartitions = new ArrayList<>(); - public PartitionCommitter(String schemaName, String tableName, HiveMetastore metastore, int batchSize) + public PartitionCommitter(ConnectorSession session, String schemaName, String tableName, HiveMetastore metastore, int batchSize) { + this.session = session; this.schemaName = schemaName; this.tableName = tableName; this.metastore = metastore; @@ -766,7 +771,7 @@ public void abort() // drop created partitions for (Partition createdPartition : getCreatedPartitions()) { try { - metastore.dropPartition(schemaName, tableName, createdPartition.getValues()); + metastore.dropPartition(session.getUser(), schemaName, tableName, createdPartition.getValues()); } catch (Exception e) { log.error(e, "Error rolling back new partition '%s' in table '%s.%s", createdPartition.getValues(), schemaName, tableName); @@ -776,7 +781,7 @@ public void abort() private void addBatch() { - metastore.addPartitions(schemaName, tableName, batch); + metastore.addPartitions(session.getUser(), schemaName, tableName, batch); createdPartitions.addAll(batch); batch.clear(); } @@ -793,11 +798,11 @@ public void commitInsert(ConnectorSession session, ConnectorInsertTableHandle in .collect(toList()); HiveStorageFormat storageFormat = handle.getHiveStorageFormat(); - PartitionCommitter partitionCommitter = new PartitionCommitter(handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); + PartitionCommitter partitionCommitter = new PartitionCommitter(session, handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); try { partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates); - Optional
table = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
table = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(handle.getSchemaName(), handle.getTableName())); } @@ -890,7 +895,7 @@ public void rollbackInsert(ConnectorSession session, ConnectorInsertTableHandle } // Otherwise, insert was directly into the target table and partitions, and all must be checked for temp files - Optional
table = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
table = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!table.isPresent()) { log.error("Error rolling back insert into table %s.%s. Table was dropped during insert, and data directory may contain temporary data", handle.getSchemaName(), handle.getTableName()); return; @@ -904,10 +909,10 @@ public void rollbackInsert(ConnectorSession session, ConnectorInsertTableHandle // check every existing partition that is outside for the base directory if (!table.get().getPartitionKeys().isEmpty()) { - List partitionNames = metastore.getPartitionNames(handle.getSchemaName(), handle.getTableName()) + List partitionNames = metastore.getPartitionNames(session.getUser(), handle.getSchemaName(), handle.getTableName()) .orElse(ImmutableList.of()); for (List partitionNameBatch : Iterables.partition(partitionNames, 10)) { - metastore.getPartitionsByNames(handle.getSchemaName(), handle.getTableName(), partitionNameBatch).orElse(ImmutableMap.of()).values().stream() + metastore.getPartitionsByNames(session.getUser(), handle.getSchemaName(), handle.getTableName(), partitionNameBatch).orElse(ImmutableMap.of()).values().stream() .map(partition -> partition.getSd().getLocation()) .filter(location -> !location.startsWith(tableDirectory)) .forEach(locationsToClean::add); @@ -1135,7 +1140,7 @@ public void createView(ConnectorSession session, SchemaTableName viewName, Strin ImmutableMap.of())); try { - metastore.createTable(table); + metastore.createTable(session.getUser(), table); } catch (TableAlreadyExistsException e) { throw new ViewAlreadyExistsException(e.getTableName()); @@ -1151,7 +1156,7 @@ public void dropView(ConnectorSession session, SchemaTableName viewName) } try { - metastore.dropTable(viewName.getSchemaName(), viewName.getTableName()); + metastore.dropTable(session.getUser(), viewName.getSchemaName(), viewName.getTableName()); } catch (TableNotFoundException e) { throw new ViewNotFoundException(e.getTableName()); @@ -1163,7 +1168,7 @@ public List listViews(ConnectorSession session, String schemaNa { ImmutableList.Builder tableNames = ImmutableList.builder(); for (String schemaName : listSchemas(session, schemaNameOrNull)) { - for (String tableName : metastore.getAllViews(schemaName).orElse(emptyList())) { + for (String tableName : metastore.getAllViews(session.getUser(), schemaName).orElse(emptyList())) { tableNames.add(new SchemaTableName(schemaName, tableName)); } } @@ -1183,7 +1188,7 @@ public Map getViews(ConnectorSession s } for (SchemaTableName schemaTableName : tableNames) { - Optional
table = metastore.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), schemaTableName.getSchemaName(), schemaTableName.getTableName()); if (table.isPresent() && HiveUtil.isPrestoView(table.get())) { views.put(schemaTableName, new ConnectorViewDefinition( schemaTableName, @@ -1215,7 +1220,7 @@ public OptionalLong metadataDelete(ConnectorSession session, ConnectorTableHandl //for (HivePartition hivePartition : layoutHandle.getOrComputePartitions(this, session, tableHandle)) { for (HivePartition hivePartition : getOrComputePartitions(layoutHandle, session, tableHandle)) { - metastore.dropPartitionByName(handle.getSchemaName(), handle.getTableName(), hivePartition.getPartitionId()); + metastore.dropPartitionByName(session.getUser(), handle.getSchemaName(), handle.getTableName(), hivePartition.getPartitionId()); } // it is too expensive to determine the exact number of deleted rows return OptionalLong.empty(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java index 4e991d1547c4..6444e05878f1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.metastore.HiveMetastore; import com.facebook.presto.spi.ConnectorPageSink; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageIndexer; import com.facebook.presto.spi.PageIndexerFactory; @@ -88,6 +89,8 @@ public class HivePageSink implements ConnectorPageSink { + private final ConnectorSession session; + private final String schemaName; private final String tableName; @@ -122,6 +125,7 @@ public class HivePageSink private HiveRecordWriter[] writers = new HiveRecordWriter[0]; public HivePageSink( + ConnectorSession session, String schemaName, String tableName, boolean isCreateTable, @@ -138,6 +142,7 @@ public HivePageSink( boolean immutablePartitions, JsonCodec partitionUpdateCodec) { + this.session = requireNonNull(session, "session is null"); this.schemaName = requireNonNull(schemaName, "schemaName is null"); this.tableName = requireNonNull(tableName, "tableName is null"); @@ -210,7 +215,7 @@ public HivePageSink( conf = new JobConf(hdfsEnvironment.getConfiguration(writePath.get())); } else { - Optional
table = metastore.getTable(schemaName, tableName); + Optional
table = metastore.getTable(session.getUser(), schemaName, tableName); if (!table.isPresent()) { throw new PrestoException(HIVE_INVALID_METADATA, format("Table %s.%s was dropped during insert", schemaName, tableName)); } @@ -287,7 +292,7 @@ private HiveRecordWriter createWriter(List partitionRow) // attempt to get the existing partition (if this is an existing partitioned table) Optional partition = Optional.empty(); if (!partitionRow.isEmpty() && table != null) { - partition = metastore.getPartition(schemaName, tableName, partitionName); + partition = metastore.getPartition(session.getUser(), schemaName, tableName, partitionName); } if (!partition.isPresent()) { @@ -303,7 +308,7 @@ private HiveRecordWriter createWriter(List partitionRow) .map(HiveType::toHiveType) .map(HiveType::getHiveTypeName) .collect(Collectors.joining(":"))); - target = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName).toString(); + target = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName).toString(); if (!partitionRow.isEmpty()) { // verify the target directory for the partition does not already exist diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java index 7cb88075d5f6..313fe5f5592d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java @@ -64,19 +64,20 @@ public HivePageSinkProvider( public ConnectorPageSink createPageSink(ConnectorSession session, ConnectorOutputTableHandle tableHandle) { HiveWritableTableHandle handle = checkType(tableHandle, HiveOutputTableHandle.class, "tableHandle"); - return createPageSink(handle, true); + return createPageSink(session, handle, true); } @Override public ConnectorPageSink createPageSink(ConnectorSession session, ConnectorInsertTableHandle tableHandle) { HiveInsertTableHandle handle = checkType(tableHandle, HiveInsertTableHandle.class, "tableHandle"); - return createPageSink(handle, false); + return createPageSink(session, handle, false); } - private ConnectorPageSink createPageSink(HiveWritableTableHandle handle, boolean isCreateTable) + private ConnectorPageSink createPageSink(ConnectorSession session, HiveWritableTableHandle handle, boolean isCreateTable) { return new HivePageSink( + session, handle.getSchemaName(), handle.getTableName(), isCreateTable, diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java index 37eb15d30025..b41f6cb702fc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java @@ -100,7 +100,7 @@ public HivePartitionResult getPartitions(ConnectorSession session, HiveMetastore } SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); - Table table = getTable(metastore, tableName); + Table table = getTable(session, metastore, tableName); Optional bucket = getHiveBucket(table, effectivePredicate.extractFixedValues()); TupleDomain compactEffectivePredicate = toCompactTupleDomain(effectivePredicate, domainCompactionThreshold); @@ -110,7 +110,7 @@ public HivePartitionResult getPartitions(ConnectorSession session, HiveMetastore } List partitionColumns = getPartitionKeyColumnHandles(connectorId, table); - List partitionNames = getFilteredPartitionNames(metastore, tableName, partitionColumns, effectivePredicate); + List partitionNames = getFilteredPartitionNames(session, metastore, tableName, partitionColumns, effectivePredicate); // do a final pass to filter based on fields that could not be used to filter the partitions ImmutableList.Builder partitions = ImmutableList.builder(); @@ -164,9 +164,9 @@ private Optional> parseValuesAndFilte return Optional.of(builder.build()); } - private Table getTable(HiveMetastore metastore, SchemaTableName tableName) + private Table getTable(ConnectorSession session, HiveMetastore metastore, SchemaTableName tableName) { - Optional
target = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
target = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } @@ -185,7 +185,7 @@ private Table getTable(HiveMetastore metastore, SchemaTableName tableName) return table; } - private List getFilteredPartitionNames(HiveMetastore metastore, SchemaTableName tableName, List partitionKeys, TupleDomain effectivePredicate) + private List getFilteredPartitionNames(ConnectorSession session, HiveMetastore metastore, SchemaTableName tableName, List partitionKeys, TupleDomain effectivePredicate) { List filter = new ArrayList<>(); for (HiveColumnHandle partitionKey : partitionKeys) { @@ -217,7 +217,7 @@ else if ((value instanceof Boolean) || (value instanceof Double) || (value insta } // fetch the partition names - return metastore.getPartitionNamesByParts(tableName.getSchemaName(), tableName.getTableName(), filter) + return metastore.getPartitionNamesByParts(session.getUser(), tableName.getSchemaName(), tableName.getTableName(), filter) .orElseThrow(() -> new TableNotFoundException(tableName)); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index 8f63692545bc..1506fc8f3c64 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -157,11 +157,11 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa // sort partitions partitions = Ordering.natural().onResultOf(HivePartition::getPartitionId).reverse().sortedCopy(partitions); - Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } - Iterable hivePartitions = getPartitionMetadata(table.get(), tableName, partitions); + Iterable hivePartitions = getPartitionMetadata(session, table.get(), tableName, partitions); HiveSplitLoader hiveSplitLoader = new BackgroundHiveSplitLoader( connectorId, @@ -185,7 +185,7 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa return splitSource; } - private Iterable getPartitionMetadata(Table table, SchemaTableName tableName, List hivePartitions) + private Iterable getPartitionMetadata(ConnectorSession session, Table table, SchemaTableName tableName, List hivePartitions) { if (hivePartitions.isEmpty()) { return ImmutableList.of(); @@ -201,6 +201,7 @@ private Iterable getPartitionMetadata(Table table, Schema Iterable> partitionNameBatches = partitionExponentially(hivePartitions, minPartitionBatchSize, maxPartitionBatchSize); Iterable> partitionBatches = transform(partitionNameBatches, partitionBatch -> { Optional> batch = metastore.getPartitionsByNames( + session.getUser(), tableName.getSchemaName(), tableName.getTableName(), Lists.transform(partitionBatch, HivePartition::getPartitionId)); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java index 58e7aa397a98..0ebaa8158583 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.metastore.HiveMetastore; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; @@ -301,9 +302,9 @@ private static void checkWritable( } } - public static Path getTableDefaultLocation(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, String schemaName, String tableName) + public static Path getTableDefaultLocation(ConnectorSession session, HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, String schemaName, String tableName) { - String location = getDatabase(metastore, schemaName).getLocationUri(); + String location = getDatabase(session, metastore, schemaName).getLocationUri(); if (isNullOrEmpty(location)) { throw new PrestoException(HIVE_DATABASE_LOCATION_ERROR, format("Database '%s' location is not set", schemaName)); } @@ -319,9 +320,9 @@ public static Path getTableDefaultLocation(HiveMetastore metastore, HdfsEnvironm return new Path(databasePath, tableName); } - private static Database getDatabase(HiveMetastore metastore, String database) + private static Database getDatabase(ConnectorSession session, HiveMetastore metastore, String database) { - return metastore.getDatabase(database).orElseThrow(() -> new SchemaNotFoundException(database)); + return metastore.getDatabase(session.getUser(), database).orElseThrow(() -> new SchemaNotFoundException(database)); } public static boolean pathExists(HdfsEnvironment hdfsEnvironment, Path path) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java index 75d7074a543a..ed668aabb338 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java @@ -305,7 +305,7 @@ private static Map getAll(LoadingCache cache, Iterable key } @Override - public List getAllDatabases() + public List getAllDatabases(String user) { return get(databaseNamesCache, ""); } @@ -328,7 +328,7 @@ private List loadAllDatabases() } @Override - public Optional getDatabase(String databaseName) + public Optional getDatabase(String user, String databaseName) { return get(databaseCache, databaseName); } @@ -355,7 +355,7 @@ private Optional loadDatabase(String databaseName) } @Override - public Optional> getAllTables(String databaseName) + public Optional> getAllTables(String user, String databaseName) { return get(tableNamesCache, databaseName); } @@ -398,13 +398,13 @@ private Optional> loadAllTables(String databaseName) } @Override - public Optional
getTable(String databaseName, String tableName) + public Optional
getTable(String user, String databaseName, String tableName) { return get(tableCache, HiveTableName.table(databaseName, tableName)); } @Override - public Optional> getAllViews(String databaseName) + public Optional> getAllViews(String user, String databaseName) { return get(viewNamesCache, databaseName); } @@ -432,7 +432,7 @@ private Optional> loadAllViews(String databaseName) } @Override - public void createTable(Table table) + public void createTable(String user, Table table) { try { retry() @@ -467,7 +467,7 @@ public void createTable(Table table) } @Override - public void dropTable(String databaseName, String tableName) + public void dropTable(String user, String databaseName, String tableName) { try { retry() @@ -506,7 +506,7 @@ protected void invalidateTable(String databaseName, String tableName) } @Override - public void alterTable(String databaseName, String tableName, Table table) + public void alterTable(String user, String databaseName, String tableName, Table table) { try { retry() @@ -570,7 +570,7 @@ private Optional
loadTable(HiveTableName hiveTableName) } @Override - public Optional> getPartitionNames(String databaseName, String tableName) + public Optional> getPartitionNames(String user, String databaseName, String tableName) { return get(partitionNamesCache, HiveTableName.table(databaseName, tableName)); } @@ -602,7 +602,7 @@ private Optional> loadPartitionNames(HiveTableName hiveTableName) } @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) { return get(partitionFilterCache, PartitionFilter.partitionFilter(databaseName, tableName, parts)); } @@ -632,7 +632,7 @@ private Optional> loadPartitionNamesByParts(PartitionFilter partiti } @Override - public void addPartitions(String databaseName, String tableName, List partitions) + public void addPartitions(String user, String databaseName, String tableName, List partitions) { if (partitions.isEmpty()) { return; @@ -676,7 +676,7 @@ public void addPartitions(String databaseName, String tableName, List } @Override - public void dropPartition(String databaseName, String tableName, List parts) + public void dropPartition(String user, String databaseName, String tableName, List parts) { try { retry() @@ -707,7 +707,7 @@ public void dropPartition(String databaseName, String tableName, List pa } @Override - public void dropPartitionByName(String databaseName, String tableName, String partitionName) + public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) { try { retry() @@ -753,7 +753,7 @@ private void invalidatePartitionCache(String databaseName, String tableName) } @Override - public Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames) + public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) { Iterable names = transform(partitionNames, name -> HivePartitionName.partition(databaseName, tableName, name)); @@ -769,7 +769,7 @@ public Optional> getPartitionsByNames(String databaseName } @Override - public Optional getPartition(String databaseName, String tableName, String partitionName) + public Optional getPartition(String user, String databaseName, String tableName, String partitionName) { HivePartitionName name = HivePartitionName.partition(databaseName, tableName, partitionName); return get(partitionCache, name); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java index 49e9b71da794..066cb9576a23 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java @@ -30,43 +30,43 @@ public interface HiveMetastore { String DEFAULT_DATABASE_NAME = "default"; - void createTable(Table table); + void createTable(String user, Table table); - void dropTable(String databaseName, String tableName); + void dropTable(String user, String databaseName, String tableName); - void alterTable(String databaseName, String tableName, Table table); + void alterTable(String user, String databaseName, String tableName, Table table); @Managed void flushCache(); - List getAllDatabases(); + List getAllDatabases(String user); - Optional> getAllTables(String databaseName); + Optional> getAllTables(String user, String databaseName); - Optional> getAllViews(String databaseName); + Optional> getAllViews(String user, String databaseName); - Optional getDatabase(String databaseName); + Optional getDatabase(String user, String databaseName); /** * Adds partitions to the table in a single atomic task. The implementation * must either add all partitions and return normally, or add no partitions and * throw an exception. */ - void addPartitions(String databaseName, String tableName, List partitions); + void addPartitions(String user, String databaseName, String tableName, List partitions); - void dropPartition(String databaseName, String tableName, List parts); + void dropPartition(String user, String databaseName, String tableName, List parts); - void dropPartitionByName(String databaseName, String tableName, String partitionName); + void dropPartitionByName(String user, String databaseName, String tableName, String partitionName); - Optional> getPartitionNames(String databaseName, String tableName); + Optional> getPartitionNames(String user, String databaseName, String tableName); - Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts); + Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts); - Optional getPartition(String databaseName, String tableName, String partitionName); + Optional getPartition(String user, String databaseName, String tableName, String partitionName); - Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames); + Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames); - Optional
getTable(String databaseName, String tableName); + Optional
getTable(String user, String databaseName, String tableName); Set getRoles(String user); @@ -81,7 +81,7 @@ default boolean isDatabaseOwner(String user, String databaseName) return true; } - Optional databaseMetadata = getDatabase(databaseName); + Optional databaseMetadata = getDatabase(user, databaseName); if (!databaseMetadata.isPresent()) { return false; } @@ -101,7 +101,7 @@ default boolean isDatabaseOwner(String user, String databaseName) default boolean isTableOwner(String user, String databaseName, String tableName) { // a table can only be owned by a user - Optional
table = getTable(databaseName, tableName); + Optional
table = getTable(user, databaseName, tableName); return table.isPresent() && user.equals(table.get().getOwner()); } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 1cd4adf8046d..8a5692904b4c 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -1564,7 +1564,7 @@ protected void doCreateEmptyTable(SchemaTableName tableName, HiveStorageFormat s assertEquals(tableMetadata.getColumns(), createTableColumns); // verify table format - Table table = getMetastoreClient(tableName.getSchemaName()).getTable(tableName.getSchemaName(), tableName.getTableName()).get(); + Table table = getMetastoreClient(tableName.getSchemaName()).getTable(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()).get(); if (!table.getSd().getInputFormat().equals(storageFormat.getInputFormat())) { assertEquals(table.getSd().getInputFormat(), storageFormat.getInputFormat()); } @@ -1678,7 +1678,7 @@ protected Set listAllDataFiles(ConnectorInsertTableHandle tableHandle) protected Set listAllDataFiles(String schemaName, String tableName) throws IOException { - Table table = metastoreClient.getTable(schemaName, tableName).get(); + Table table = metastoreClient.getTable(SESSION.getUser(), schemaName, tableName).get(); Path path = new Path(table.getSd().getLocation()); Set existingFiles = new HashSet<>(); return listAllDataFiles(path, existingFiles); @@ -1715,7 +1715,7 @@ private void doInsertPartitioned(HiveStorageFormat storageFormat, SchemaTableNam insertData(tableHandle, CREATE_TABLE_PARTITIONED_DATA, SESSION); // verify partitions were created - List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) + List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) @@ -1800,7 +1800,7 @@ private void doMetadataDelete(HiveStorageFormat storageFormat, SchemaTableName t } // verify partitions were created - List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) + List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java index 1bb81269e532..7869d77b9035 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java @@ -325,7 +325,7 @@ private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageF // table, which fails without explicit configuration for S3. // We work around that by using a dummy location when creating the // table and update it here to the correct S3 location. - metastoreClient.updateTableLocation(database, tableName.getTableName(), outputHandle.getWritePath().get()); + metastoreClient.updateTableLocation(SESSION.getUser(), database, tableName.getTableName(), outputHandle.getWritePath().get()); // load the new table ConnectorTableHandle tableHandle = getTableHandle(tableName); @@ -352,7 +352,7 @@ private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageF private void dropTable(SchemaTableName table) { try { - metastoreClient.dropTable(table.getSchemaName(), table.getTableName()); + metastoreClient.dropTable(SESSION.getUser(), table.getSchemaName(), table.getTableName()); } catch (RuntimeException e) { // this usually occurs because the table was not created @@ -402,9 +402,9 @@ public TestingHiveMetastore(HiveCluster hiveCluster, ExecutorService executor, H } @Override - public Optional getDatabase(String databaseName) + public Optional getDatabase(String user, String databaseName) { - Optional database = super.getDatabase(databaseName); + Optional database = super.getDatabase(user, databaseName); if (database.isPresent()) { database.get().setLocationUri("s3://" + writableBucket + "/"); } @@ -412,18 +412,18 @@ public Optional getDatabase(String databaseName) } @Override - public void createTable(Table table) + public void createTable(String user, Table table) { // hack to work around the metastore not being configured for S3 table.getSd().setLocation("/"); - super.createTable(table); + super.createTable(user, table); } @Override - public void dropTable(String databaseName, String tableName) + public void dropTable(String user, String databaseName, String tableName) { try { - Optional
table = getTable(databaseName, tableName); + Optional
table = getTable(user, databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } @@ -449,10 +449,10 @@ public void dropTable(String databaseName, String tableName) } } - public void updateTableLocation(String databaseName, String tableName, String location) + public void updateTableLocation(String user, String databaseName, String tableName, String location) { try { - Optional
table = getTable(databaseName, tableName); + Optional
table = getTable(user, databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java index fa8a81344c20..56bb639c7f33 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java @@ -88,13 +88,13 @@ public void createDatabase(Database database) } @Override - public List getAllDatabases() + public List getAllDatabases(String user) { return ImmutableList.copyOf(databases.keySet()); } @Override - public void createTable(Table table) + public void createTable(String user1, Table table) { SchemaTableName schemaTableName = new SchemaTableName(table.getDbName(), table.getTableName()); Table tableCopy = table.deepCopy(); @@ -137,7 +137,7 @@ else if (tableCopy.getSd().getLocation() != null) { } @Override - public void dropTable(String databaseName, String tableName) + public void dropTable(String user, String databaseName, String tableName) { SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); Table table = relations.remove(schemaTableName); @@ -159,7 +159,7 @@ public void dropTable(String databaseName, String tableName) } @Override - public void alterTable(String databaseName, String tableName, Table newTable) + public void alterTable(String user, String databaseName, String tableName, Table newTable) { SchemaTableName oldName = new SchemaTableName(databaseName, tableName); SchemaTableName newName = new SchemaTableName(newTable.getDbName(), newTable.getTableName()); @@ -186,7 +186,7 @@ public void alterTable(String databaseName, String tableName, Table newTable) } @Override - public Optional> getAllTables(String databaseName) + public Optional> getAllTables(String user, String databaseName) { ImmutableList.Builder tables = ImmutableList.builder(); for (SchemaTableName schemaTableName : this.relations.keySet()) { @@ -198,7 +198,7 @@ public Optional> getAllTables(String databaseName) } @Override - public Optional> getAllViews(String databaseName) + public Optional> getAllViews(String user, String databaseName) { ImmutableList.Builder tables = ImmutableList.builder(); for (SchemaTableName schemaTableName : this.views.keySet()) { @@ -210,15 +210,15 @@ public Optional> getAllViews(String databaseName) } @Override - public Optional getDatabase(String databaseName) + public Optional getDatabase(String user, String databaseName) { return Optional.ofNullable(databases.get(databaseName)); } @Override - public void addPartitions(String databaseName, String tableName, List partitions) + public void addPartitions(String user, String databaseName, String tableName, List partitions) { - Optional
table = getTable(databaseName, tableName); + Optional
table = getTable(user, databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } @@ -233,7 +233,7 @@ public void addPartitions(String databaseName, String tableName, List } @Override - public void dropPartition(String databaseName, String tableName, List parts) + public void dropPartition(String user, String databaseName, String tableName, List parts) { for (Entry entry : partitions.entrySet()) { PartitionName partitionName = entry.getKey(); @@ -245,7 +245,7 @@ public void dropPartition(String databaseName, String tableName, List pa } @Override - public void dropPartitionByName(String databaseName, String tableName, String partitionName) + public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) { for (PartitionName partition : partitions.keySet()) { if (partition.matches(databaseName, tableName, partitionName)) { @@ -255,7 +255,7 @@ public void dropPartitionByName(String databaseName, String tableName, String pa } @Override - public Optional> getPartitionNames(String databaseName, String tableName) + public Optional> getPartitionNames(String user, String databaseName, String tableName) { return Optional.of(ImmutableList.copyOf(partitions.entrySet().stream() .filter(entry -> entry.getKey().matches(databaseName, tableName)) @@ -264,7 +264,7 @@ public Optional> getPartitionNames(String databaseName, String tabl } @Override - public Optional getPartition(String databaseName, String tableName, String partitionName) + public Optional getPartition(String user, String databaseName, String tableName, String partitionName) { PartitionName name = new PartitionName(databaseName, tableName, partitionName); Partition partition = partitions.get(name); @@ -275,7 +275,7 @@ public Optional getPartition(String databaseName, String tableName, S } @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) { return Optional.of(partitions.entrySet().stream() .filter(entry -> partitionMatches(entry.getValue(), databaseName, tableName, parts)) @@ -303,7 +303,7 @@ private static boolean partitionMatches(Partition partition, String databaseName } @Override - public Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames) + public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) { ImmutableMap.Builder builder = ImmutableMap.builder(); for (String name : partitionNames) { @@ -318,7 +318,7 @@ public Optional> getPartitionsByNames(String databaseName } @Override - public Optional
getTable(String databaseName, String tableName) + public Optional
getTable(String user, String databaseName, String tableName) { SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); return Optional.ofNullable(relations.get(schemaTableName)); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java index b0643d13a7e8..ac55e00933fa 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; +import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.BAD_DATABASE; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.TEST_DATABASE; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.TEST_PARTITION1; @@ -55,14 +56,14 @@ public void testGetAllDatabases() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 2); } @@ -71,21 +72,21 @@ public void testGetAllTable() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); } public void testInvalidDbGetAllTAbles() throws Exception { - assertFalse(metastore.getAllTables(BAD_DATABASE).isPresent()); + assertFalse(metastore.getAllTables(SESSION.getUser(), BAD_DATABASE).isPresent()); } @Test @@ -93,21 +94,21 @@ public void testGetTable() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); } public void testInvalidDbGetTable() throws Exception { - assertFalse(metastore.getTable(BAD_DATABASE, TEST_TABLE).isPresent()); + assertFalse(metastore.getTable(SESSION.getUser(), BAD_DATABASE, TEST_TABLE).isPresent()); } @Test @@ -116,14 +117,14 @@ public void testGetPartitionNames() { ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } @@ -131,7 +132,7 @@ public void testGetPartitionNames() public void testInvalidGetPartitionNames() throws Exception { - assertEquals(metastore.getPartitionNames(BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); + assertEquals(metastore.getPartitionNames(SESSION.getUser(), BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); } @Test @@ -142,14 +143,14 @@ public void testGetPartitionNamesByParts() ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } @@ -157,7 +158,7 @@ public void testInvalidGetPartitionNamesByParts() throws Exception { ImmutableList parts = ImmutableList.of(); - assertFalse(metastore.getPartitionNamesByParts(BAD_DATABASE, TEST_TABLE, parts).isPresent()); + assertFalse(metastore.getPartitionNamesByParts(SESSION.getUser(), BAD_DATABASE, TEST_TABLE, parts).isPresent()); } @Test @@ -165,35 +166,35 @@ public void testGetPartitionsByNames() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - metastore.getTable(TEST_DATABASE, TEST_TABLE); + metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE); assertEquals(mockClient.getAccessCount(), 1); // Select half of the available partitions and load them into the cache - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); assertEquals(mockClient.getAccessCount(), 2); // Now select all of the partitions - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); // There should be one more access to fetch the remaining partition assertEquals(mockClient.getAccessCount(), 3); // Now if we fetch any or both of them, they should not hit the client - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).get().size(), 1); - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); assertEquals(mockClient.getAccessCount(), 3); metastore.flushCache(); // Fetching both should only result in one batched access - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); assertEquals(mockClient.getAccessCount(), 4); } public void testInvalidGetPartitionsByNames() throws Exception { - assertFalse(metastore.getPartitionsByNames(BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).isPresent()); + assertFalse(metastore.getPartitionsByNames(SESSION.getUser(), BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).isPresent()); } @Test @@ -203,7 +204,7 @@ public void testNoCacheExceptions() // Throw exceptions on usage mockClient.setThrowException(true); try { - metastore.getAllDatabases(); + metastore.getAllDatabases(SESSION.getUser()); } catch (RuntimeException ignored) { } @@ -211,7 +212,7 @@ public void testNoCacheExceptions() // Second try should hit the client again try { - metastore.getAllDatabases(); + metastore.getAllDatabases(SESSION.getUser()); } catch (RuntimeException ignored) { } From bae9de697343a0fbdd15e01b7fb43a0f2583a7a6 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 13:23:31 -0700 Subject: [PATCH 14/83] Revert 972d5d166bc8174c9cef50b1066f6efc9a07a02b --- .../parquet/ParquetRecordCursorProvider.java | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java index 3854449498da..eb6bb880bed3 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java @@ -27,12 +27,10 @@ import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; import org.joda.time.DateTimeZone; import javax.inject.Inject; -import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -90,26 +88,20 @@ public Optional createHiveRecordCursor( throw new IllegalArgumentException("Can not read Parquet column: " + unsupportedColumns); } - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); - try { - return ugi.doAs((PrivilegedExceptionAction>) () -> Optional.of(new ParquetHiveRecordCursor( - configuration, - path, - start, - length, - schema, - partitionKeys, - columns, - useParquetColumnNames, - hiveStorageTimeZone, - typeManager, - isParquetPredicatePushdownEnabled(session), - effectivePredicate - ))); - } - catch (Exception e) { - throw new RuntimeException(e); - } + return Optional.of(new ParquetHiveRecordCursor( + configuration, + path, + start, + length, + schema, + partitionKeys, + columns, + useParquetColumnNames, + hiveStorageTimeZone, + typeManager, + isParquetPredicatePushdownEnabled(session), + effectivePredicate + )); } private static Predicate isParquetSupportedType() From a39315881019902b237d11540219a1c94b7ecc0a Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 13:24:09 -0700 Subject: [PATCH 15/83] Load splits as user so that hdfs reads happen as that user. --- .../com/facebook/presto/hive/BackgroundHiveSplitLoader.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 82defdecc369..557544a96b6b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -38,8 +38,10 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collections; import java.util.Deque; @@ -162,8 +164,9 @@ public TaskStatus process() try { CompletableFuture future; taskExecutionLock.readLock().lock(); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); try { - future = loadSplits(); + future = ugi.doAs((PrivilegedExceptionAction>) BackgroundHiveSplitLoader.this::loadSplits); } finally { taskExecutionLock.readLock().unlock(); From 1b65e2fad9ac91da587e0193b114933461b7fcb6 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 13:26:36 -0700 Subject: [PATCH 16/83] UserBasedHiveMetastore to access metastore as user issuing the query. --- .../presto/hive/HiveClientModule.java | 6 +- .../hive/ThriftHiveMetastoreClient.java | 5 + .../metastore/UserBasedHiveMetastore.java | 608 ++++++++++++++++++ .../UserBasedHiveMetastoreStats.java | 141 ++++ 4 files changed, 757 insertions(+), 3 deletions(-) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index bc48cbfc7f0f..de95fec6e6fb 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; -import com.facebook.presto.hive.metastore.CachingHiveMetastore; import com.facebook.presto.hive.metastore.HiveMetastore; +import com.facebook.presto.hive.metastore.UserBasedHiveMetastore; import com.facebook.presto.hive.orc.DwrfPageSourceFactory; import com.facebook.presto.hive.orc.DwrfRecordCursorProvider; import com.facebook.presto.hive.orc.OrcPageSourceFactory; @@ -81,9 +81,9 @@ public void configure(Binder binder) binder.bind(HiveMetastore.class).toInstance(metastore); } else { - binder.bind(HiveMetastore.class).to(CachingHiveMetastore.class).in(Scopes.SINGLETON); + binder.bind(HiveMetastore.class).to(UserBasedHiveMetastore.class).in(Scopes.SINGLETON); newExporter(binder).export(HiveMetastore.class) - .as(generatedNameOf(CachingHiveMetastore.class, connectorId)); + .as(generatedNameOf(UserBasedHiveMetastore.class, connectorId)); } binder.bind(NamenodeStats.class).in(Scopes.SINGLETON); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java b/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java index d97aaf3d464f..9ab9986b7bab 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java @@ -49,6 +49,11 @@ public ThriftHiveMetastoreClient(TProtocol protocol) this.client = new ThriftHiveMetastore.Client(protocol); } + public ThriftHiveMetastore.Client getClient() + { + return client; + } + @Override public void close() { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java new file mode 100644 index 000000000000..251962a01cc9 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java @@ -0,0 +1,608 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore; + +import com.facebook.presto.hive.HiveCluster; +import com.facebook.presto.hive.HiveViewNotSupportedException; +import com.facebook.presto.hive.TableAlreadyExistsException; +import com.facebook.presto.hive.ThriftHiveMetastoreClient; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.SchemaNotFoundException; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableNotFoundException; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.HiveObjectRef; +import org.apache.hadoop.hive.metastore.api.HiveObjectType; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; +import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; +import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.thrift.TException; +import org.weakref.jmx.Flatten; +import org.weakref.jmx.Managed; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.function.Function; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; +import static com.facebook.presto.hive.HiveUtil.PRESTO_VIEW_FLAG; +import static com.facebook.presto.hive.HiveUtil.isPrestoView; +import static com.facebook.presto.hive.RetryDriver.retry; +import static com.facebook.presto.hive.metastore.HivePrivilege.OWNERSHIP; +import static com.facebook.presto.hive.metastore.HivePrivilege.parsePrivilege; +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toSet; +import static org.apache.hadoop.hive.metastore.api.PrincipalType.USER; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS; + +public class UserBasedHiveMetastore + implements HiveMetastore +{ + private final UserBasedHiveMetastoreStats stats = new UserBasedHiveMetastoreStats(); + protected final HiveCluster clientProvider; + + @Inject + public UserBasedHiveMetastore(HiveCluster hiveCluster) + { + this.clientProvider = requireNonNull(hiveCluster, "hiveCluster is null"); + } + + @Managed + @Flatten + public UserBasedHiveMetastoreStats getStats() + { + return stats; + } + + protected Function getExceptionMapper() + { + return Function.identity(); + } + + @Override + public void createTable(String user, Table table) + { + try { + retry() + .exceptionMapper(getExceptionMapper()) + .stopOn(AlreadyExistsException.class, InvalidObjectException.class, MetaException.class, NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("createTable", stats.getCreateTable().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + client.createTable(table); + } + return null; + })); + } + catch (AlreadyExistsException e) { + throw new TableAlreadyExistsException(new SchemaTableName(table.getDbName(), table.getTableName())); + } + catch (NoSuchObjectException e) { + throw new SchemaNotFoundException(table.getDbName()); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw Throwables.propagate(e); + } + } + + @Override + public void dropTable(String user, String databaseName, String tableName) + { + try { + retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("dropTable", stats.getDropTable().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + client.dropTable(databaseName, tableName, true); + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public void alterTable(String user, String databaseName, String tableName, Table table) + { + try { + retry() + .exceptionMapper(getExceptionMapper()) + .stopOn(InvalidOperationException.class, MetaException.class) + .stopOnIllegalExceptions() + .run("alterTable", stats.getAlterTable().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + Optional
source = getTable(user, databaseName, tableName); + if (!source.isPresent()) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + client.getClient().set_ugi(user, ImmutableList.of()); + client.alterTable(databaseName, tableName, table); + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (InvalidOperationException | MetaException e) { + throw com.google.common.base.Throwables.propagate(e); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public void flushCache() + { + } + + @Override + public List getAllDatabases(String user) + { + try { + return retry() + .stopOnIllegalExceptions() + .run("getAllDatabases", stats.getGetAllDatabases().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return client.getAllDatabases(); + } + })); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional> getAllTables(String user, String databaseName) + { + Callable> getAllTables = stats.getGetAllTables().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return client.getAllTables(databaseName); + } + }); + + Callable getDatabase = stats.getGetDatabase().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + client.getDatabase(databaseName); + return null; + } + }); + + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getAllTables", () -> { + List tables = getAllTables.call(); + if (tables.isEmpty()) { + // Check to see if the database exists + getDatabase.call(); + } + return Optional.of(tables); + }); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional> getAllViews(String user, String databaseName) + { + try { + return retry() + .stopOn(UnknownDBException.class) + .stopOnIllegalExceptions() + .run("getAllViews", stats.getAllViews().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + String filter = HIVE_FILTER_FIELD_PARAMS + PRESTO_VIEW_FLAG + " = \"true\""; + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getTableNamesByFilter(databaseName, filter)); + } + })); + } + catch (UnknownDBException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional getDatabase(String user, String databaseName) + { + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getDatabase", stats.getGetDatabase().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getDatabase(databaseName)); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public void addPartitions(String user, String databaseName, String tableName, List partitions) + { + if (partitions.isEmpty()) { + return; + } + try { + retry() + .exceptionMapper(getExceptionMapper()) + .stopOn(org.apache.hadoop.hive.metastore.api.AlreadyExistsException.class, org.apache.hadoop.hive.metastore.api.InvalidObjectException.class, MetaException.class, org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, PrestoException.class) + .stopOnIllegalExceptions() + .run("addPartitions", stats.getAddPartitions().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + int partitionsAdded = client.addPartitions(partitions); + if (partitionsAdded != partitions.size()) { + throw new PrestoException(HIVE_METASTORE_ERROR, + format("Hive metastore only added %s of %s partitions", partitionsAdded, partitions.size())); + } + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.AlreadyExistsException | org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + // todo partition already exists exception + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public void dropPartition(String user, String databaseName, String tableName, List parts) + { + try { + retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, MetaException.class) + .stopOnIllegalExceptions() + .run("dropPartition", stats.getDropPartition().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + client.dropPartition(databaseName, tableName, parts, true); + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) + { + try { + retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, MetaException.class) + .stopOnIllegalExceptions() + .run("dropPartitionByName", stats.getDropPartitionByName().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + // It is observed that: (examples below assumes a table with one partition column `ds`) + // * When a partition doesn't exist (e.g. ds=2015-09-99), this thrift call is a no-op. It doesn't throw any exception. + // * When a typo exists in partition column name (e.g. dxs=2015-09-01), this thrift call will delete ds=2015-09-01. + client.getClient().set_ugi(user, ImmutableList.of()); + client.dropPartitionByName(databaseName, tableName, partitionName, true); + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public Optional> getPartitionNames(String user, String databaseName, String tableName) + { + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getPartitionNames", stats.getGetPartitionNames().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getPartitionNames(databaseName, tableName)); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) + { + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getPartitionNamesByParts", stats.getGetPartitionNamesPs().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getPartitionNamesFiltered(databaseName, tableName, parts)); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional getPartition(String user, String databaseName, String tableName, String partitionName) + { + requireNonNull(partitionName, "partitionName is null"); + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getPartitionsByNames", stats.getGetPartitionByName().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getPartitionByName(databaseName, tableName, partitionName)); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) + { + requireNonNull(partitionNames, "partitionNames is null"); + checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty"); + + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getPartitionsByNames", stats.getGetPartitionsByNames().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + List partitionColumnNames = ImmutableList.copyOf(Warehouse.makeSpecFromName(partitionNames.get(0)).keySet()); + ImmutableMap.Builder partitions = ImmutableMap.builder(); + client.getClient().set_ugi(user, ImmutableList.of()); + for (Partition partition : client.getPartitionsByNames(databaseName, tableName, partitionNames)) { + String partitionId = FileUtils.makePartName(partitionColumnNames, partition.getValues(), null); + partitions.put(partitionId, partition); + } + return Optional.of(partitions.build()); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + // assume none of the partitions in the batch are available + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional
getTable(String user, String databaseName, String tableName) + { + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, HiveViewNotSupportedException.class) + .stopOnIllegalExceptions() + .run("getTable", stats.getGetTable().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + Table table = client.getTable(databaseName, tableName); + if (table.getTableType().equals(TableType.VIRTUAL_VIEW.name()) && (!isPrestoView(table))) { + throw new HiveViewNotSupportedException(new SchemaTableName(databaseName, tableName)); + } + return Optional.of(table); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Set getRoles(String user) + { + try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { + List roles = client.listRoles(user, USER); + if (roles == null) { + return ImmutableSet.of(); + } + return ImmutableSet.copyOf(roles.stream() + .map(Role::getRoleName) + .collect(toSet())); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Set getDatabasePrivileges(String user, String databaseName) + { + ImmutableSet.Builder privileges = ImmutableSet.builder(); + + if (isDatabaseOwner(user, databaseName)) { + privileges.add(OWNERSHIP); + } + privileges.addAll(getPrivileges(user, new HiveObjectRef(HiveObjectType.DATABASE, databaseName, null, null, null))); + + return privileges.build(); + } + + @Override + public Set getTablePrivileges(String user, String databaseName, String tableName) + { + ImmutableSet.Builder privileges = ImmutableSet.builder(); + + if (isTableOwner(user, databaseName, tableName)) { + privileges.add(OWNERSHIP); + } + privileges.addAll(getPrivileges(user, new HiveObjectRef(HiveObjectType.TABLE, databaseName, tableName, null, null))); + + return privileges.build(); + } + + private Set getPrivileges(String user, HiveObjectRef objectReference) + { + ImmutableSet.Builder privileges = ImmutableSet.builder(); + try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { + PrincipalPrivilegeSet privilegeSet = client.getPrivilegeSet(objectReference, user, null); + + if (privilegeSet != null) { + Map> userPrivileges = privilegeSet.getUserPrivileges(); + if (userPrivileges != null) { + privileges.addAll(toGrants(userPrivileges.get(user))); + } + for (List rolePrivileges : privilegeSet.getRolePrivileges().values()) { + privileges.addAll(toGrants(rolePrivileges)); + } + // We do not add the group permissions as Hive does not seem to process these + } + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + + return privileges.build(); + } + + private static Set toGrants(List userGrants) + { + if (userGrants == null) { + return ImmutableSet.of(); + } + + ImmutableSet.Builder privileges = ImmutableSet.builder(); + for (PrivilegeGrantInfo userGrant : userGrants) { + privileges.addAll(parsePrivilege(userGrant)); + if (userGrant.isGrantOption()) { + privileges.add(HivePrivilege.GRANT); + } + } + return privileges.build(); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java new file mode 100644 index 000000000000..7778e2ba9010 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java @@ -0,0 +1,141 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore; + +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +public class UserBasedHiveMetastoreStats +{ + private final HiveMetastoreApiStats getAllDatabases = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getDatabase = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getAllTables = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getAllViews = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getTable = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getPartitionNames = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getPartitionNamesPs = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getPartitionByName = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getPartitionsByNames = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats createTable = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats dropTable = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats alterTable = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats addPartitions = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats dropPartition = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats dropPartitionByName = new HiveMetastoreApiStats(); + + @Managed + @Nested + public HiveMetastoreApiStats getGetAllDatabases() + { + return getAllDatabases; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetDatabase() + { + return getDatabase; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetAllTables() + { + return getAllTables; + } + + @Managed + @Nested + public HiveMetastoreApiStats getAllViews() + { + return getAllViews; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetTable() + { + return getTable; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetPartitionNames() + { + return getPartitionNames; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetPartitionNamesPs() + { + return getPartitionNamesPs; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetPartitionByName() + { + return getPartitionByName; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetPartitionsByNames() + { + return getPartitionsByNames; + } + + @Managed + @Nested + public HiveMetastoreApiStats getCreateTable() + { + return createTable; + } + + @Managed + @Nested + public HiveMetastoreApiStats getDropTable() + { + return dropTable; + } + + @Managed + @Nested + public HiveMetastoreApiStats getAlterTable() + { + return alterTable; + } + + @Managed + @Nested + public HiveMetastoreApiStats getAddPartitions() + { + return addPartitions; + } + + @Managed + @Nested + public HiveMetastoreApiStats getDropPartition() + { + return dropPartition; + } + + @Managed + @Nested + public HiveMetastoreApiStats getDropPartitionByName() + { + return dropPartitionByName; + } +} From 7cbc53a635331388b4fcfd6601b4cffc88aa8cbb Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 15:03:42 -0700 Subject: [PATCH 17/83] Import missed import. From acfd0c75cc96fe1f4e289a69866b32bef538010c Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 15:43:54 -0700 Subject: [PATCH 18/83] Use standard charset for UTF-8 instead of string. From a5a4f8cdaec61eb5172c9b7680780c46e3aba484 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Tue, 3 Nov 2015 12:08:31 -0800 Subject: [PATCH 19/83] Pick username from unix system and disallow overriding it. --- .../main/java/com/facebook/presto/cli/ClientOptions.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java b/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java index 32bf33079643..364958cd6065 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java @@ -17,6 +17,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.common.net.HostAndPort; +import com.sun.security.auth.module.UnixSystem; import io.airlift.airline.Option; import io.airlift.http.client.spnego.KerberosConfig; @@ -66,8 +67,9 @@ public class ClientOptions @Option(name = "--keystore-password", title = "keystore password", description = "Keystore password") public String keystorePassword; - @Option(name = "--user", title = "user", description = "Username") - public String user = System.getProperty("user.name"); + // Pick the user name for the logged in user. + // Do not let it be overridden by users. + public String user = new UnixSystem().getUsername(); @Option(name = "--source", title = "source", description = "Name of source making query") public String source = "presto-cli"; From e26fd783db6a07310faddf73470f83c3e1d6e845 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 23 Nov 2015 10:53:27 -0800 Subject: [PATCH 20/83] Revert "Propagate user session information throughout hive metastore." This reverts commit d01400182645e82f733760a2d938273c49dd5a88. --- .../facebook/presto/hive/HiveMetadata.java | 85 +++++++++---------- .../facebook/presto/hive/HivePageSink.java | 11 +-- .../presto/hive/HivePageSinkProvider.java | 7 +- .../presto/hive/HivePartitionManager.java | 12 +-- .../presto/hive/HiveSplitManager.java | 7 +- .../facebook/presto/hive/HiveWriteUtils.java | 9 +- .../hive/metastore/CachingHiveMetastore.java | 30 +++---- .../presto/hive/metastore/HiveMetastore.java | 34 ++++---- .../presto/hive/AbstractTestHiveClient.java | 8 +- .../presto/hive/AbstractTestHiveClientS3.java | 20 ++--- .../hive/metastore/InMemoryHiveMetastore.java | 32 +++---- .../metastore/TestCachingHiveMetastore.java | 59 +++++++------ 12 files changed, 150 insertions(+), 164 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index a3ad7180b9de..c8e48aa7e21b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -231,14 +231,14 @@ public HivePartitionManager getPartitionManager() @Override public List listSchemaNames(ConnectorSession session) { - return metastore.getAllDatabases(session.getUser()); + return metastore.getAllDatabases(); } @Override public HiveTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { requireNonNull(tableName, "tableName is null"); - if (!metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()).isPresent()) { + if (!metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).isPresent()) { return null; } return new HiveTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName()); @@ -249,12 +249,12 @@ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, Connect { requireNonNull(tableHandle, "tableHandle is null"); SchemaTableName tableName = schemaTableName(tableHandle); - return getTableMetadata(session, tableName); + return getTableMetadata(tableName); } - private ConnectorTableMetadata getTableMetadata(ConnectorSession session, SchemaTableName tableName) + private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) { - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) { throw new TableNotFoundException(tableName); } @@ -294,7 +294,7 @@ public List listTables(ConnectorSession session, String schemaN { ImmutableList.Builder tableNames = ImmutableList.builder(); for (String schemaName : listSchemas(session, schemaNameOrNull)) { - for (String tableName : metastore.getAllTables(session.getUser(), schemaName).orElse(emptyList())) { + for (String tableName : metastore.getAllTables(schemaName).orElse(emptyList())) { tableNames.add(new SchemaTableName(schemaName, tableName)); } } @@ -313,7 +313,7 @@ private List listSchemas(ConnectorSession session, String schemaNameOrNu public ColumnHandle getSampleWeightColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -335,7 +335,7 @@ public boolean canCreateSampledTables(ConnectorSession session) public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -356,7 +356,7 @@ public Map> listTableColumns(ConnectorSess ImmutableMap.Builder> columns = ImmutableMap.builder(); for (SchemaTableName tableName : listTables(session, prefix)) { try { - columns.put(tableName, getTableMetadata(session, tableName).getColumns()); + columns.put(tableName, getTableMetadata(tableName).getColumns()); } catch (HiveViewNotSupportedException e) { // view is not supported @@ -397,18 +397,17 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe List partitionedBy = getPartitionedBy(tableMetadata.getProperties()); List columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); - createTable(session, schemaName, tableName, tableMetadata.getOwner(), columnHandles, hiveStorageFormat, partitionedBy); + createTable(schemaName, tableName, tableMetadata.getOwner(), columnHandles, hiveStorageFormat, partitionedBy); } - public void createTable(ConnectorSession session, - String schemaName, + public void createTable(String schemaName, String tableName, String tableOwner, List columnHandles, HiveStorageFormat hiveStorageFormat, List partitionedBy) { - Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName); + Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName); // verify the target directory for the table if (pathExists(hdfsEnvironment, targetPath)) { @@ -416,11 +415,10 @@ public void createTable(ConnectorSession session, } createDirectory(hdfsEnvironment, targetPath); - createTable(session, schemaName, tableName, tableOwner, columnHandles, hiveStorageFormat, partitionedBy, targetPath); + createTable(schemaName, tableName, tableOwner, columnHandles, hiveStorageFormat, partitionedBy, targetPath); } - private Table createTable(ConnectorSession session, - String schemaName, + private Table createTable(String schemaName, String tableName, String tableOwner, List columnHandles, @@ -486,7 +484,7 @@ else if (!partitionColumnNames.contains(name)) { ImmutableMap.of(), ImmutableMap.of())); - metastore.createTable(session.getUser(), table); + metastore.createTable(table); return table; } @@ -498,7 +496,7 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle } HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); - Optional
tableMetadata = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
tableMetadata = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(handle.getSchemaTableName()); } @@ -511,7 +509,7 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle sd.setCols(columns.build()); table.setSd(sd); - metastore.alterTable(session.getUser(), handle.getSchemaName(), handle.getTableName(), table); + metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); } @Override @@ -523,7 +521,7 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHan HiveTableHandle hiveTableHandle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); HiveColumnHandle sourceHandle = checkType(source, HiveColumnHandle.class, "columnHandle"); - Optional
tableMetadata = metastore.getTable(session.getUser(), hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); + Optional
tableMetadata = metastore.getTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(hiveTableHandle.getSchemaTableName()); } @@ -540,7 +538,7 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHan } sd.setCols(columns.build()); table.setSd(sd); - metastore.alterTable(session.getUser(), hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); + metastore.alterTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); } @Override @@ -552,14 +550,14 @@ public void renameTable(ConnectorSession session, ConnectorTableHandle tableHand HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); SchemaTableName tableName = schemaTableName(tableHandle); - Optional
source = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
source = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!source.isPresent()) { throw new TableNotFoundException(tableName); } Table table = source.get(); table.setDbName(newTableName.getSchemaName()); table.setTableName(newTableName.getTableName()); - metastore.alterTable(session.getUser(), handle.getSchemaName(), handle.getTableName(), table); + metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); } @Override @@ -572,7 +570,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle throw new PrestoException(PERMISSION_DENIED, "DROP TABLE is disabled in this Hive catalog"); } - Optional
target = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
target = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } @@ -581,7 +579,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle if (!session.getUser().equals(table.getOwner())) { throw new PrestoException(PERMISSION_DENIED, format("Unable to drop table '%s': owner of the table is different from session user", table)); } - metastore.dropTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + metastore.dropTable(handle.getSchemaName(), handle.getTableName()); } @Override @@ -601,7 +599,7 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto List columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); - Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName); + Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName); // verify the target directory for the table if (pathExists(hdfsEnvironment, targetPath)) { @@ -635,7 +633,7 @@ public void commitCreateTable(ConnectorSession session, ConnectorOutputTableHand .map(partitionUpdateCodec::fromJson) .collect(toList()); - Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, handle.getSchemaName(), handle.getTableName()); + Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, handle.getSchemaName(), handle.getTableName()); Path writePath = new Path(handle.getWritePath().get()); // rename if using a temporary directory @@ -651,12 +649,11 @@ public void commitCreateTable(ConnectorSession session, ConnectorOutputTableHand renameDirectory(hdfsEnvironment, handle.getSchemaName(), handle.getTableName(), writePath, targetPath); } - PartitionCommitter partitionCommitter = new PartitionCommitter(session, handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); + PartitionCommitter partitionCommitter = new PartitionCommitter(handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); try { partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates); Table table = createTable( - session, handle.getSchemaName(), handle.getTableName(), handle.getTableOwner(), @@ -694,7 +691,7 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl verifyJvmTimeZone(); SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -727,7 +724,6 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl private static class PartitionCommitter implements Closeable { - private final ConnectorSession session; private final String schemaName; private final String tableName; private final HiveMetastore metastore; @@ -735,9 +731,8 @@ private static class PartitionCommitter private final List batch; private final List createdPartitions = new ArrayList<>(); - public PartitionCommitter(ConnectorSession session, String schemaName, String tableName, HiveMetastore metastore, int batchSize) + public PartitionCommitter(String schemaName, String tableName, HiveMetastore metastore, int batchSize) { - this.session = session; this.schemaName = schemaName; this.tableName = tableName; this.metastore = metastore; @@ -771,7 +766,7 @@ public void abort() // drop created partitions for (Partition createdPartition : getCreatedPartitions()) { try { - metastore.dropPartition(session.getUser(), schemaName, tableName, createdPartition.getValues()); + metastore.dropPartition(schemaName, tableName, createdPartition.getValues()); } catch (Exception e) { log.error(e, "Error rolling back new partition '%s' in table '%s.%s", createdPartition.getValues(), schemaName, tableName); @@ -781,7 +776,7 @@ public void abort() private void addBatch() { - metastore.addPartitions(session.getUser(), schemaName, tableName, batch); + metastore.addPartitions(schemaName, tableName, batch); createdPartitions.addAll(batch); batch.clear(); } @@ -798,11 +793,11 @@ public void commitInsert(ConnectorSession session, ConnectorInsertTableHandle in .collect(toList()); HiveStorageFormat storageFormat = handle.getHiveStorageFormat(); - PartitionCommitter partitionCommitter = new PartitionCommitter(session, handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); + PartitionCommitter partitionCommitter = new PartitionCommitter(handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); try { partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates); - Optional
table = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
table = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(handle.getSchemaName(), handle.getTableName())); } @@ -895,7 +890,7 @@ public void rollbackInsert(ConnectorSession session, ConnectorInsertTableHandle } // Otherwise, insert was directly into the target table and partitions, and all must be checked for temp files - Optional
table = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
table = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!table.isPresent()) { log.error("Error rolling back insert into table %s.%s. Table was dropped during insert, and data directory may contain temporary data", handle.getSchemaName(), handle.getTableName()); return; @@ -909,10 +904,10 @@ public void rollbackInsert(ConnectorSession session, ConnectorInsertTableHandle // check every existing partition that is outside for the base directory if (!table.get().getPartitionKeys().isEmpty()) { - List partitionNames = metastore.getPartitionNames(session.getUser(), handle.getSchemaName(), handle.getTableName()) + List partitionNames = metastore.getPartitionNames(handle.getSchemaName(), handle.getTableName()) .orElse(ImmutableList.of()); for (List partitionNameBatch : Iterables.partition(partitionNames, 10)) { - metastore.getPartitionsByNames(session.getUser(), handle.getSchemaName(), handle.getTableName(), partitionNameBatch).orElse(ImmutableMap.of()).values().stream() + metastore.getPartitionsByNames(handle.getSchemaName(), handle.getTableName(), partitionNameBatch).orElse(ImmutableMap.of()).values().stream() .map(partition -> partition.getSd().getLocation()) .filter(location -> !location.startsWith(tableDirectory)) .forEach(locationsToClean::add); @@ -1140,7 +1135,7 @@ public void createView(ConnectorSession session, SchemaTableName viewName, Strin ImmutableMap.of())); try { - metastore.createTable(session.getUser(), table); + metastore.createTable(table); } catch (TableAlreadyExistsException e) { throw new ViewAlreadyExistsException(e.getTableName()); @@ -1156,7 +1151,7 @@ public void dropView(ConnectorSession session, SchemaTableName viewName) } try { - metastore.dropTable(session.getUser(), viewName.getSchemaName(), viewName.getTableName()); + metastore.dropTable(viewName.getSchemaName(), viewName.getTableName()); } catch (TableNotFoundException e) { throw new ViewNotFoundException(e.getTableName()); @@ -1168,7 +1163,7 @@ public List listViews(ConnectorSession session, String schemaNa { ImmutableList.Builder tableNames = ImmutableList.builder(); for (String schemaName : listSchemas(session, schemaNameOrNull)) { - for (String tableName : metastore.getAllViews(session.getUser(), schemaName).orElse(emptyList())) { + for (String tableName : metastore.getAllViews(schemaName).orElse(emptyList())) { tableNames.add(new SchemaTableName(schemaName, tableName)); } } @@ -1188,7 +1183,7 @@ public Map getViews(ConnectorSession s } for (SchemaTableName schemaTableName : tableNames) { - Optional
table = metastore.getTable(session.getUser(), schemaTableName.getSchemaName(), schemaTableName.getTableName()); + Optional
table = metastore.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()); if (table.isPresent() && HiveUtil.isPrestoView(table.get())) { views.put(schemaTableName, new ConnectorViewDefinition( schemaTableName, @@ -1220,7 +1215,7 @@ public OptionalLong metadataDelete(ConnectorSession session, ConnectorTableHandl //for (HivePartition hivePartition : layoutHandle.getOrComputePartitions(this, session, tableHandle)) { for (HivePartition hivePartition : getOrComputePartitions(layoutHandle, session, tableHandle)) { - metastore.dropPartitionByName(session.getUser(), handle.getSchemaName(), handle.getTableName(), hivePartition.getPartitionId()); + metastore.dropPartitionByName(handle.getSchemaName(), handle.getTableName(), hivePartition.getPartitionId()); } // it is too expensive to determine the exact number of deleted rows return OptionalLong.empty(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java index 6444e05878f1..4e991d1547c4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java @@ -15,7 +15,6 @@ import com.facebook.presto.hive.metastore.HiveMetastore; import com.facebook.presto.spi.ConnectorPageSink; -import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageIndexer; import com.facebook.presto.spi.PageIndexerFactory; @@ -89,8 +88,6 @@ public class HivePageSink implements ConnectorPageSink { - private final ConnectorSession session; - private final String schemaName; private final String tableName; @@ -125,7 +122,6 @@ public class HivePageSink private HiveRecordWriter[] writers = new HiveRecordWriter[0]; public HivePageSink( - ConnectorSession session, String schemaName, String tableName, boolean isCreateTable, @@ -142,7 +138,6 @@ public HivePageSink( boolean immutablePartitions, JsonCodec partitionUpdateCodec) { - this.session = requireNonNull(session, "session is null"); this.schemaName = requireNonNull(schemaName, "schemaName is null"); this.tableName = requireNonNull(tableName, "tableName is null"); @@ -215,7 +210,7 @@ public HivePageSink( conf = new JobConf(hdfsEnvironment.getConfiguration(writePath.get())); } else { - Optional
table = metastore.getTable(session.getUser(), schemaName, tableName); + Optional
table = metastore.getTable(schemaName, tableName); if (!table.isPresent()) { throw new PrestoException(HIVE_INVALID_METADATA, format("Table %s.%s was dropped during insert", schemaName, tableName)); } @@ -292,7 +287,7 @@ private HiveRecordWriter createWriter(List partitionRow) // attempt to get the existing partition (if this is an existing partitioned table) Optional partition = Optional.empty(); if (!partitionRow.isEmpty() && table != null) { - partition = metastore.getPartition(session.getUser(), schemaName, tableName, partitionName); + partition = metastore.getPartition(schemaName, tableName, partitionName); } if (!partition.isPresent()) { @@ -308,7 +303,7 @@ private HiveRecordWriter createWriter(List partitionRow) .map(HiveType::toHiveType) .map(HiveType::getHiveTypeName) .collect(Collectors.joining(":"))); - target = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName).toString(); + target = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName).toString(); if (!partitionRow.isEmpty()) { // verify the target directory for the partition does not already exist diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java index 313fe5f5592d..7cb88075d5f6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java @@ -64,20 +64,19 @@ public HivePageSinkProvider( public ConnectorPageSink createPageSink(ConnectorSession session, ConnectorOutputTableHandle tableHandle) { HiveWritableTableHandle handle = checkType(tableHandle, HiveOutputTableHandle.class, "tableHandle"); - return createPageSink(session, handle, true); + return createPageSink(handle, true); } @Override public ConnectorPageSink createPageSink(ConnectorSession session, ConnectorInsertTableHandle tableHandle) { HiveInsertTableHandle handle = checkType(tableHandle, HiveInsertTableHandle.class, "tableHandle"); - return createPageSink(session, handle, false); + return createPageSink(handle, false); } - private ConnectorPageSink createPageSink(ConnectorSession session, HiveWritableTableHandle handle, boolean isCreateTable) + private ConnectorPageSink createPageSink(HiveWritableTableHandle handle, boolean isCreateTable) { return new HivePageSink( - session, handle.getSchemaName(), handle.getTableName(), isCreateTable, diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java index b41f6cb702fc..37eb15d30025 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java @@ -100,7 +100,7 @@ public HivePartitionResult getPartitions(ConnectorSession session, HiveMetastore } SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); - Table table = getTable(session, metastore, tableName); + Table table = getTable(metastore, tableName); Optional bucket = getHiveBucket(table, effectivePredicate.extractFixedValues()); TupleDomain compactEffectivePredicate = toCompactTupleDomain(effectivePredicate, domainCompactionThreshold); @@ -110,7 +110,7 @@ public HivePartitionResult getPartitions(ConnectorSession session, HiveMetastore } List partitionColumns = getPartitionKeyColumnHandles(connectorId, table); - List partitionNames = getFilteredPartitionNames(session, metastore, tableName, partitionColumns, effectivePredicate); + List partitionNames = getFilteredPartitionNames(metastore, tableName, partitionColumns, effectivePredicate); // do a final pass to filter based on fields that could not be used to filter the partitions ImmutableList.Builder partitions = ImmutableList.builder(); @@ -164,9 +164,9 @@ private Optional> parseValuesAndFilte return Optional.of(builder.build()); } - private Table getTable(ConnectorSession session, HiveMetastore metastore, SchemaTableName tableName) + private Table getTable(HiveMetastore metastore, SchemaTableName tableName) { - Optional
target = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
target = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } @@ -185,7 +185,7 @@ private Table getTable(ConnectorSession session, HiveMetastore metastore, Schema return table; } - private List getFilteredPartitionNames(ConnectorSession session, HiveMetastore metastore, SchemaTableName tableName, List partitionKeys, TupleDomain effectivePredicate) + private List getFilteredPartitionNames(HiveMetastore metastore, SchemaTableName tableName, List partitionKeys, TupleDomain effectivePredicate) { List filter = new ArrayList<>(); for (HiveColumnHandle partitionKey : partitionKeys) { @@ -217,7 +217,7 @@ else if ((value instanceof Boolean) || (value instanceof Double) || (value insta } // fetch the partition names - return metastore.getPartitionNamesByParts(session.getUser(), tableName.getSchemaName(), tableName.getTableName(), filter) + return metastore.getPartitionNamesByParts(tableName.getSchemaName(), tableName.getTableName(), filter) .orElseThrow(() -> new TableNotFoundException(tableName)); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index 1506fc8f3c64..8f63692545bc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -157,11 +157,11 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa // sort partitions partitions = Ordering.natural().onResultOf(HivePartition::getPartitionId).reverse().sortedCopy(partitions); - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } - Iterable hivePartitions = getPartitionMetadata(session, table.get(), tableName, partitions); + Iterable hivePartitions = getPartitionMetadata(table.get(), tableName, partitions); HiveSplitLoader hiveSplitLoader = new BackgroundHiveSplitLoader( connectorId, @@ -185,7 +185,7 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa return splitSource; } - private Iterable getPartitionMetadata(ConnectorSession session, Table table, SchemaTableName tableName, List hivePartitions) + private Iterable getPartitionMetadata(Table table, SchemaTableName tableName, List hivePartitions) { if (hivePartitions.isEmpty()) { return ImmutableList.of(); @@ -201,7 +201,6 @@ private Iterable getPartitionMetadata(ConnectorSession se Iterable> partitionNameBatches = partitionExponentially(hivePartitions, minPartitionBatchSize, maxPartitionBatchSize); Iterable> partitionBatches = transform(partitionNameBatches, partitionBatch -> { Optional> batch = metastore.getPartitionsByNames( - session.getUser(), tableName.getSchemaName(), tableName.getTableName(), Lists.transform(partitionBatch, HivePartition::getPartitionId)); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java index 0ebaa8158583..58e7aa397a98 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java @@ -14,7 +14,6 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.metastore.HiveMetastore; -import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; @@ -302,9 +301,9 @@ private static void checkWritable( } } - public static Path getTableDefaultLocation(ConnectorSession session, HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, String schemaName, String tableName) + public static Path getTableDefaultLocation(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, String schemaName, String tableName) { - String location = getDatabase(session, metastore, schemaName).getLocationUri(); + String location = getDatabase(metastore, schemaName).getLocationUri(); if (isNullOrEmpty(location)) { throw new PrestoException(HIVE_DATABASE_LOCATION_ERROR, format("Database '%s' location is not set", schemaName)); } @@ -320,9 +319,9 @@ public static Path getTableDefaultLocation(ConnectorSession session, HiveMetasto return new Path(databasePath, tableName); } - private static Database getDatabase(ConnectorSession session, HiveMetastore metastore, String database) + private static Database getDatabase(HiveMetastore metastore, String database) { - return metastore.getDatabase(session.getUser(), database).orElseThrow(() -> new SchemaNotFoundException(database)); + return metastore.getDatabase(database).orElseThrow(() -> new SchemaNotFoundException(database)); } public static boolean pathExists(HdfsEnvironment hdfsEnvironment, Path path) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java index ed668aabb338..75d7074a543a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java @@ -305,7 +305,7 @@ private static Map getAll(LoadingCache cache, Iterable key } @Override - public List getAllDatabases(String user) + public List getAllDatabases() { return get(databaseNamesCache, ""); } @@ -328,7 +328,7 @@ private List loadAllDatabases() } @Override - public Optional getDatabase(String user, String databaseName) + public Optional getDatabase(String databaseName) { return get(databaseCache, databaseName); } @@ -355,7 +355,7 @@ private Optional loadDatabase(String databaseName) } @Override - public Optional> getAllTables(String user, String databaseName) + public Optional> getAllTables(String databaseName) { return get(tableNamesCache, databaseName); } @@ -398,13 +398,13 @@ private Optional> loadAllTables(String databaseName) } @Override - public Optional
getTable(String user, String databaseName, String tableName) + public Optional
getTable(String databaseName, String tableName) { return get(tableCache, HiveTableName.table(databaseName, tableName)); } @Override - public Optional> getAllViews(String user, String databaseName) + public Optional> getAllViews(String databaseName) { return get(viewNamesCache, databaseName); } @@ -432,7 +432,7 @@ private Optional> loadAllViews(String databaseName) } @Override - public void createTable(String user, Table table) + public void createTable(Table table) { try { retry() @@ -467,7 +467,7 @@ public void createTable(String user, Table table) } @Override - public void dropTable(String user, String databaseName, String tableName) + public void dropTable(String databaseName, String tableName) { try { retry() @@ -506,7 +506,7 @@ protected void invalidateTable(String databaseName, String tableName) } @Override - public void alterTable(String user, String databaseName, String tableName, Table table) + public void alterTable(String databaseName, String tableName, Table table) { try { retry() @@ -570,7 +570,7 @@ private Optional
loadTable(HiveTableName hiveTableName) } @Override - public Optional> getPartitionNames(String user, String databaseName, String tableName) + public Optional> getPartitionNames(String databaseName, String tableName) { return get(partitionNamesCache, HiveTableName.table(databaseName, tableName)); } @@ -602,7 +602,7 @@ private Optional> loadPartitionNames(HiveTableName hiveTableName) } @Override - public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) + public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) { return get(partitionFilterCache, PartitionFilter.partitionFilter(databaseName, tableName, parts)); } @@ -632,7 +632,7 @@ private Optional> loadPartitionNamesByParts(PartitionFilter partiti } @Override - public void addPartitions(String user, String databaseName, String tableName, List partitions) + public void addPartitions(String databaseName, String tableName, List partitions) { if (partitions.isEmpty()) { return; @@ -676,7 +676,7 @@ public void addPartitions(String user, String databaseName, String tableName, Li } @Override - public void dropPartition(String user, String databaseName, String tableName, List parts) + public void dropPartition(String databaseName, String tableName, List parts) { try { retry() @@ -707,7 +707,7 @@ public void dropPartition(String user, String databaseName, String tableName, Li } @Override - public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) + public void dropPartitionByName(String databaseName, String tableName, String partitionName) { try { retry() @@ -753,7 +753,7 @@ private void invalidatePartitionCache(String databaseName, String tableName) } @Override - public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) + public Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames) { Iterable names = transform(partitionNames, name -> HivePartitionName.partition(databaseName, tableName, name)); @@ -769,7 +769,7 @@ public Optional> getPartitionsByNames(String user, String } @Override - public Optional getPartition(String user, String databaseName, String tableName, String partitionName) + public Optional getPartition(String databaseName, String tableName, String partitionName) { HivePartitionName name = HivePartitionName.partition(databaseName, tableName, partitionName); return get(partitionCache, name); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java index 066cb9576a23..49e9b71da794 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java @@ -30,43 +30,43 @@ public interface HiveMetastore { String DEFAULT_DATABASE_NAME = "default"; - void createTable(String user, Table table); + void createTable(Table table); - void dropTable(String user, String databaseName, String tableName); + void dropTable(String databaseName, String tableName); - void alterTable(String user, String databaseName, String tableName, Table table); + void alterTable(String databaseName, String tableName, Table table); @Managed void flushCache(); - List getAllDatabases(String user); + List getAllDatabases(); - Optional> getAllTables(String user, String databaseName); + Optional> getAllTables(String databaseName); - Optional> getAllViews(String user, String databaseName); + Optional> getAllViews(String databaseName); - Optional getDatabase(String user, String databaseName); + Optional getDatabase(String databaseName); /** * Adds partitions to the table in a single atomic task. The implementation * must either add all partitions and return normally, or add no partitions and * throw an exception. */ - void addPartitions(String user, String databaseName, String tableName, List partitions); + void addPartitions(String databaseName, String tableName, List partitions); - void dropPartition(String user, String databaseName, String tableName, List parts); + void dropPartition(String databaseName, String tableName, List parts); - void dropPartitionByName(String user, String databaseName, String tableName, String partitionName); + void dropPartitionByName(String databaseName, String tableName, String partitionName); - Optional> getPartitionNames(String user, String databaseName, String tableName); + Optional> getPartitionNames(String databaseName, String tableName); - Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts); + Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts); - Optional getPartition(String user, String databaseName, String tableName, String partitionName); + Optional getPartition(String databaseName, String tableName, String partitionName); - Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames); + Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames); - Optional
getTable(String user, String databaseName, String tableName); + Optional
getTable(String databaseName, String tableName); Set getRoles(String user); @@ -81,7 +81,7 @@ default boolean isDatabaseOwner(String user, String databaseName) return true; } - Optional databaseMetadata = getDatabase(user, databaseName); + Optional databaseMetadata = getDatabase(databaseName); if (!databaseMetadata.isPresent()) { return false; } @@ -101,7 +101,7 @@ default boolean isDatabaseOwner(String user, String databaseName) default boolean isTableOwner(String user, String databaseName, String tableName) { // a table can only be owned by a user - Optional
table = getTable(user, databaseName, tableName); + Optional
table = getTable(databaseName, tableName); return table.isPresent() && user.equals(table.get().getOwner()); } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 8a5692904b4c..1cd4adf8046d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -1564,7 +1564,7 @@ protected void doCreateEmptyTable(SchemaTableName tableName, HiveStorageFormat s assertEquals(tableMetadata.getColumns(), createTableColumns); // verify table format - Table table = getMetastoreClient(tableName.getSchemaName()).getTable(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()).get(); + Table table = getMetastoreClient(tableName.getSchemaName()).getTable(tableName.getSchemaName(), tableName.getTableName()).get(); if (!table.getSd().getInputFormat().equals(storageFormat.getInputFormat())) { assertEquals(table.getSd().getInputFormat(), storageFormat.getInputFormat()); } @@ -1678,7 +1678,7 @@ protected Set listAllDataFiles(ConnectorInsertTableHandle tableHandle) protected Set listAllDataFiles(String schemaName, String tableName) throws IOException { - Table table = metastoreClient.getTable(SESSION.getUser(), schemaName, tableName).get(); + Table table = metastoreClient.getTable(schemaName, tableName).get(); Path path = new Path(table.getSd().getLocation()); Set existingFiles = new HashSet<>(); return listAllDataFiles(path, existingFiles); @@ -1715,7 +1715,7 @@ private void doInsertPartitioned(HiveStorageFormat storageFormat, SchemaTableNam insertData(tableHandle, CREATE_TABLE_PARTITIONED_DATA, SESSION); // verify partitions were created - List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()) + List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) @@ -1800,7 +1800,7 @@ private void doMetadataDelete(HiveStorageFormat storageFormat, SchemaTableName t } // verify partitions were created - List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()) + List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java index 7869d77b9035..1bb81269e532 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java @@ -325,7 +325,7 @@ private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageF // table, which fails without explicit configuration for S3. // We work around that by using a dummy location when creating the // table and update it here to the correct S3 location. - metastoreClient.updateTableLocation(SESSION.getUser(), database, tableName.getTableName(), outputHandle.getWritePath().get()); + metastoreClient.updateTableLocation(database, tableName.getTableName(), outputHandle.getWritePath().get()); // load the new table ConnectorTableHandle tableHandle = getTableHandle(tableName); @@ -352,7 +352,7 @@ private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageF private void dropTable(SchemaTableName table) { try { - metastoreClient.dropTable(SESSION.getUser(), table.getSchemaName(), table.getTableName()); + metastoreClient.dropTable(table.getSchemaName(), table.getTableName()); } catch (RuntimeException e) { // this usually occurs because the table was not created @@ -402,9 +402,9 @@ public TestingHiveMetastore(HiveCluster hiveCluster, ExecutorService executor, H } @Override - public Optional getDatabase(String user, String databaseName) + public Optional getDatabase(String databaseName) { - Optional database = super.getDatabase(user, databaseName); + Optional database = super.getDatabase(databaseName); if (database.isPresent()) { database.get().setLocationUri("s3://" + writableBucket + "/"); } @@ -412,18 +412,18 @@ public Optional getDatabase(String user, String databaseName) } @Override - public void createTable(String user, Table table) + public void createTable(Table table) { // hack to work around the metastore not being configured for S3 table.getSd().setLocation("/"); - super.createTable(user, table); + super.createTable(table); } @Override - public void dropTable(String user, String databaseName, String tableName) + public void dropTable(String databaseName, String tableName) { try { - Optional
table = getTable(user, databaseName, tableName); + Optional
table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } @@ -449,10 +449,10 @@ public void dropTable(String user, String databaseName, String tableName) } } - public void updateTableLocation(String user, String databaseName, String tableName, String location) + public void updateTableLocation(String databaseName, String tableName, String location) { try { - Optional
table = getTable(user, databaseName, tableName); + Optional
table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java index 56bb639c7f33..fa8a81344c20 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java @@ -88,13 +88,13 @@ public void createDatabase(Database database) } @Override - public List getAllDatabases(String user) + public List getAllDatabases() { return ImmutableList.copyOf(databases.keySet()); } @Override - public void createTable(String user1, Table table) + public void createTable(Table table) { SchemaTableName schemaTableName = new SchemaTableName(table.getDbName(), table.getTableName()); Table tableCopy = table.deepCopy(); @@ -137,7 +137,7 @@ else if (tableCopy.getSd().getLocation() != null) { } @Override - public void dropTable(String user, String databaseName, String tableName) + public void dropTable(String databaseName, String tableName) { SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); Table table = relations.remove(schemaTableName); @@ -159,7 +159,7 @@ public void dropTable(String user, String databaseName, String tableName) } @Override - public void alterTable(String user, String databaseName, String tableName, Table newTable) + public void alterTable(String databaseName, String tableName, Table newTable) { SchemaTableName oldName = new SchemaTableName(databaseName, tableName); SchemaTableName newName = new SchemaTableName(newTable.getDbName(), newTable.getTableName()); @@ -186,7 +186,7 @@ public void alterTable(String user, String databaseName, String tableName, Table } @Override - public Optional> getAllTables(String user, String databaseName) + public Optional> getAllTables(String databaseName) { ImmutableList.Builder tables = ImmutableList.builder(); for (SchemaTableName schemaTableName : this.relations.keySet()) { @@ -198,7 +198,7 @@ public Optional> getAllTables(String user, String databaseName) } @Override - public Optional> getAllViews(String user, String databaseName) + public Optional> getAllViews(String databaseName) { ImmutableList.Builder tables = ImmutableList.builder(); for (SchemaTableName schemaTableName : this.views.keySet()) { @@ -210,15 +210,15 @@ public Optional> getAllViews(String user, String databaseName) } @Override - public Optional getDatabase(String user, String databaseName) + public Optional getDatabase(String databaseName) { return Optional.ofNullable(databases.get(databaseName)); } @Override - public void addPartitions(String user, String databaseName, String tableName, List partitions) + public void addPartitions(String databaseName, String tableName, List partitions) { - Optional
table = getTable(user, databaseName, tableName); + Optional
table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } @@ -233,7 +233,7 @@ public void addPartitions(String user, String databaseName, String tableName, Li } @Override - public void dropPartition(String user, String databaseName, String tableName, List parts) + public void dropPartition(String databaseName, String tableName, List parts) { for (Entry entry : partitions.entrySet()) { PartitionName partitionName = entry.getKey(); @@ -245,7 +245,7 @@ public void dropPartition(String user, String databaseName, String tableName, Li } @Override - public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) + public void dropPartitionByName(String databaseName, String tableName, String partitionName) { for (PartitionName partition : partitions.keySet()) { if (partition.matches(databaseName, tableName, partitionName)) { @@ -255,7 +255,7 @@ public void dropPartitionByName(String user, String databaseName, String tableNa } @Override - public Optional> getPartitionNames(String user, String databaseName, String tableName) + public Optional> getPartitionNames(String databaseName, String tableName) { return Optional.of(ImmutableList.copyOf(partitions.entrySet().stream() .filter(entry -> entry.getKey().matches(databaseName, tableName)) @@ -264,7 +264,7 @@ public Optional> getPartitionNames(String user, String databaseName } @Override - public Optional getPartition(String user, String databaseName, String tableName, String partitionName) + public Optional getPartition(String databaseName, String tableName, String partitionName) { PartitionName name = new PartitionName(databaseName, tableName, partitionName); Partition partition = partitions.get(name); @@ -275,7 +275,7 @@ public Optional getPartition(String user, String databaseName, String } @Override - public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) + public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) { return Optional.of(partitions.entrySet().stream() .filter(entry -> partitionMatches(entry.getValue(), databaseName, tableName, parts)) @@ -303,7 +303,7 @@ private static boolean partitionMatches(Partition partition, String databaseName } @Override - public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) + public Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames) { ImmutableMap.Builder builder = ImmutableMap.builder(); for (String name : partitionNames) { @@ -318,7 +318,7 @@ public Optional> getPartitionsByNames(String user, String } @Override - public Optional
getTable(String user, String databaseName, String tableName) + public Optional
getTable(String databaseName, String tableName) { SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); return Optional.ofNullable(relations.get(schemaTableName)); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java index ac55e00933fa..b0643d13a7e8 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java @@ -22,7 +22,6 @@ import java.util.concurrent.TimeUnit; -import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.BAD_DATABASE; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.TEST_DATABASE; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.TEST_PARTITION1; @@ -56,14 +55,14 @@ public void testGetAllDatabases() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 2); } @@ -72,21 +71,21 @@ public void testGetAllTable() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); } public void testInvalidDbGetAllTAbles() throws Exception { - assertFalse(metastore.getAllTables(SESSION.getUser(), BAD_DATABASE).isPresent()); + assertFalse(metastore.getAllTables(BAD_DATABASE).isPresent()); } @Test @@ -94,21 +93,21 @@ public void testGetTable() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); } public void testInvalidDbGetTable() throws Exception { - assertFalse(metastore.getTable(SESSION.getUser(), BAD_DATABASE, TEST_TABLE).isPresent()); + assertFalse(metastore.getTable(BAD_DATABASE, TEST_TABLE).isPresent()); } @Test @@ -117,14 +116,14 @@ public void testGetPartitionNames() { ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } @@ -132,7 +131,7 @@ public void testGetPartitionNames() public void testInvalidGetPartitionNames() throws Exception { - assertEquals(metastore.getPartitionNames(SESSION.getUser(), BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); + assertEquals(metastore.getPartitionNames(BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); } @Test @@ -143,14 +142,14 @@ public void testGetPartitionNamesByParts() ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } @@ -158,7 +157,7 @@ public void testInvalidGetPartitionNamesByParts() throws Exception { ImmutableList parts = ImmutableList.of(); - assertFalse(metastore.getPartitionNamesByParts(SESSION.getUser(), BAD_DATABASE, TEST_TABLE, parts).isPresent()); + assertFalse(metastore.getPartitionNamesByParts(BAD_DATABASE, TEST_TABLE, parts).isPresent()); } @Test @@ -166,35 +165,35 @@ public void testGetPartitionsByNames() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE); + metastore.getTable(TEST_DATABASE, TEST_TABLE); assertEquals(mockClient.getAccessCount(), 1); // Select half of the available partitions and load them into the cache - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); assertEquals(mockClient.getAccessCount(), 2); // Now select all of the partitions - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); // There should be one more access to fetch the remaining partition assertEquals(mockClient.getAccessCount(), 3); // Now if we fetch any or both of them, they should not hit the client - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).get().size(), 1); - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); assertEquals(mockClient.getAccessCount(), 3); metastore.flushCache(); // Fetching both should only result in one batched access - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); assertEquals(mockClient.getAccessCount(), 4); } public void testInvalidGetPartitionsByNames() throws Exception { - assertFalse(metastore.getPartitionsByNames(SESSION.getUser(), BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).isPresent()); + assertFalse(metastore.getPartitionsByNames(BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).isPresent()); } @Test @@ -204,7 +203,7 @@ public void testNoCacheExceptions() // Throw exceptions on usage mockClient.setThrowException(true); try { - metastore.getAllDatabases(SESSION.getUser()); + metastore.getAllDatabases(); } catch (RuntimeException ignored) { } @@ -212,7 +211,7 @@ public void testNoCacheExceptions() // Second try should hit the client again try { - metastore.getAllDatabases(SESSION.getUser()); + metastore.getAllDatabases(); } catch (RuntimeException ignored) { } From 8c99eebea149686c123cd1a160d2e8bea9a00ca9 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 23 Nov 2015 10:54:04 -0800 Subject: [PATCH 21/83] Revert "Load splits as user so that hdfs reads happen as that user." This reverts commit a39315881019902b237d11540219a1c94b7ecc0a. --- .../com/facebook/presto/hive/BackgroundHiveSplitLoader.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 557544a96b6b..82defdecc369 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -38,10 +38,8 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collections; import java.util.Deque; @@ -164,9 +162,8 @@ public TaskStatus process() try { CompletableFuture future; taskExecutionLock.readLock().lock(); - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); try { - future = ugi.doAs((PrivilegedExceptionAction>) BackgroundHiveSplitLoader.this::loadSplits); + future = loadSplits(); } finally { taskExecutionLock.readLock().unlock(); From 884902f5dfd41f6be39454bc1c90afee9d4a3f3b Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 23 Nov 2015 10:54:41 -0800 Subject: [PATCH 22/83] Revert "UserBasedHiveMetastore to access metastore as user issuing the query." This reverts commit 1b65e2fad9ac91da587e0193b114933461b7fcb6. --- .../presto/hive/HiveClientModule.java | 6 +- .../hive/ThriftHiveMetastoreClient.java | 5 - .../metastore/UserBasedHiveMetastore.java | 608 ------------------ .../UserBasedHiveMetastoreStats.java | 141 ---- 4 files changed, 3 insertions(+), 757 deletions(-) delete mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java delete mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index de95fec6e6fb..bc48cbfc7f0f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.CachingHiveMetastore; import com.facebook.presto.hive.metastore.HiveMetastore; -import com.facebook.presto.hive.metastore.UserBasedHiveMetastore; import com.facebook.presto.hive.orc.DwrfPageSourceFactory; import com.facebook.presto.hive.orc.DwrfRecordCursorProvider; import com.facebook.presto.hive.orc.OrcPageSourceFactory; @@ -81,9 +81,9 @@ public void configure(Binder binder) binder.bind(HiveMetastore.class).toInstance(metastore); } else { - binder.bind(HiveMetastore.class).to(UserBasedHiveMetastore.class).in(Scopes.SINGLETON); + binder.bind(HiveMetastore.class).to(CachingHiveMetastore.class).in(Scopes.SINGLETON); newExporter(binder).export(HiveMetastore.class) - .as(generatedNameOf(UserBasedHiveMetastore.class, connectorId)); + .as(generatedNameOf(CachingHiveMetastore.class, connectorId)); } binder.bind(NamenodeStats.class).in(Scopes.SINGLETON); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java b/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java index 9ab9986b7bab..d97aaf3d464f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java @@ -49,11 +49,6 @@ public ThriftHiveMetastoreClient(TProtocol protocol) this.client = new ThriftHiveMetastore.Client(protocol); } - public ThriftHiveMetastore.Client getClient() - { - return client; - } - @Override public void close() { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java deleted file mode 100644 index 251962a01cc9..000000000000 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java +++ /dev/null @@ -1,608 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.hive.metastore; - -import com.facebook.presto.hive.HiveCluster; -import com.facebook.presto.hive.HiveViewNotSupportedException; -import com.facebook.presto.hive.TableAlreadyExistsException; -import com.facebook.presto.hive.ThriftHiveMetastoreClient; -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.SchemaNotFoundException; -import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.TableNotFoundException; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.HiveObjectRef; -import org.apache.hadoop.hive.metastore.api.HiveObjectType; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; -import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; -import org.apache.hadoop.hive.metastore.api.Role; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.UnknownDBException; -import org.apache.thrift.TException; -import org.weakref.jmx.Flatten; -import org.weakref.jmx.Managed; - -import javax.inject.Inject; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.function.Function; - -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; -import static com.facebook.presto.hive.HiveUtil.PRESTO_VIEW_FLAG; -import static com.facebook.presto.hive.HiveUtil.isPrestoView; -import static com.facebook.presto.hive.RetryDriver.retry; -import static com.facebook.presto.hive.metastore.HivePrivilege.OWNERSHIP; -import static com.facebook.presto.hive.metastore.HivePrivilege.parsePrivilege; -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toSet; -import static org.apache.hadoop.hive.metastore.api.PrincipalType.USER; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS; - -public class UserBasedHiveMetastore - implements HiveMetastore -{ - private final UserBasedHiveMetastoreStats stats = new UserBasedHiveMetastoreStats(); - protected final HiveCluster clientProvider; - - @Inject - public UserBasedHiveMetastore(HiveCluster hiveCluster) - { - this.clientProvider = requireNonNull(hiveCluster, "hiveCluster is null"); - } - - @Managed - @Flatten - public UserBasedHiveMetastoreStats getStats() - { - return stats; - } - - protected Function getExceptionMapper() - { - return Function.identity(); - } - - @Override - public void createTable(String user, Table table) - { - try { - retry() - .exceptionMapper(getExceptionMapper()) - .stopOn(AlreadyExistsException.class, InvalidObjectException.class, MetaException.class, NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("createTable", stats.getCreateTable().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - client.createTable(table); - } - return null; - })); - } - catch (AlreadyExistsException e) { - throw new TableAlreadyExistsException(new SchemaTableName(table.getDbName(), table.getTableName())); - } - catch (NoSuchObjectException e) { - throw new SchemaNotFoundException(table.getDbName()); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw Throwables.propagate(e); - } - } - - @Override - public void dropTable(String user, String databaseName, String tableName) - { - try { - retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("dropTable", stats.getDropTable().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - client.dropTable(databaseName, tableName, true); - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public void alterTable(String user, String databaseName, String tableName, Table table) - { - try { - retry() - .exceptionMapper(getExceptionMapper()) - .stopOn(InvalidOperationException.class, MetaException.class) - .stopOnIllegalExceptions() - .run("alterTable", stats.getAlterTable().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - Optional
source = getTable(user, databaseName, tableName); - if (!source.isPresent()) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - client.getClient().set_ugi(user, ImmutableList.of()); - client.alterTable(databaseName, tableName, table); - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (InvalidOperationException | MetaException e) { - throw com.google.common.base.Throwables.propagate(e); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public void flushCache() - { - } - - @Override - public List getAllDatabases(String user) - { - try { - return retry() - .stopOnIllegalExceptions() - .run("getAllDatabases", stats.getGetAllDatabases().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return client.getAllDatabases(); - } - })); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional> getAllTables(String user, String databaseName) - { - Callable> getAllTables = stats.getGetAllTables().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return client.getAllTables(databaseName); - } - }); - - Callable getDatabase = stats.getGetDatabase().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - client.getDatabase(databaseName); - return null; - } - }); - - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getAllTables", () -> { - List tables = getAllTables.call(); - if (tables.isEmpty()) { - // Check to see if the database exists - getDatabase.call(); - } - return Optional.of(tables); - }); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional> getAllViews(String user, String databaseName) - { - try { - return retry() - .stopOn(UnknownDBException.class) - .stopOnIllegalExceptions() - .run("getAllViews", stats.getAllViews().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - String filter = HIVE_FILTER_FIELD_PARAMS + PRESTO_VIEW_FLAG + " = \"true\""; - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getTableNamesByFilter(databaseName, filter)); - } - })); - } - catch (UnknownDBException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional getDatabase(String user, String databaseName) - { - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getDatabase", stats.getGetDatabase().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getDatabase(databaseName)); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public void addPartitions(String user, String databaseName, String tableName, List partitions) - { - if (partitions.isEmpty()) { - return; - } - try { - retry() - .exceptionMapper(getExceptionMapper()) - .stopOn(org.apache.hadoop.hive.metastore.api.AlreadyExistsException.class, org.apache.hadoop.hive.metastore.api.InvalidObjectException.class, MetaException.class, org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, PrestoException.class) - .stopOnIllegalExceptions() - .run("addPartitions", stats.getAddPartitions().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - int partitionsAdded = client.addPartitions(partitions); - if (partitionsAdded != partitions.size()) { - throw new PrestoException(HIVE_METASTORE_ERROR, - format("Hive metastore only added %s of %s partitions", partitionsAdded, partitions.size())); - } - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.AlreadyExistsException | org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - // todo partition already exists exception - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public void dropPartition(String user, String databaseName, String tableName, List parts) - { - try { - retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, MetaException.class) - .stopOnIllegalExceptions() - .run("dropPartition", stats.getDropPartition().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - client.dropPartition(databaseName, tableName, parts, true); - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) - { - try { - retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, MetaException.class) - .stopOnIllegalExceptions() - .run("dropPartitionByName", stats.getDropPartitionByName().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - // It is observed that: (examples below assumes a table with one partition column `ds`) - // * When a partition doesn't exist (e.g. ds=2015-09-99), this thrift call is a no-op. It doesn't throw any exception. - // * When a typo exists in partition column name (e.g. dxs=2015-09-01), this thrift call will delete ds=2015-09-01. - client.getClient().set_ugi(user, ImmutableList.of()); - client.dropPartitionByName(databaseName, tableName, partitionName, true); - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public Optional> getPartitionNames(String user, String databaseName, String tableName) - { - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getPartitionNames", stats.getGetPartitionNames().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getPartitionNames(databaseName, tableName)); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) - { - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getPartitionNamesByParts", stats.getGetPartitionNamesPs().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getPartitionNamesFiltered(databaseName, tableName, parts)); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional getPartition(String user, String databaseName, String tableName, String partitionName) - { - requireNonNull(partitionName, "partitionName is null"); - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getPartitionsByNames", stats.getGetPartitionByName().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getPartitionByName(databaseName, tableName, partitionName)); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) - { - requireNonNull(partitionNames, "partitionNames is null"); - checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty"); - - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getPartitionsByNames", stats.getGetPartitionsByNames().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - List partitionColumnNames = ImmutableList.copyOf(Warehouse.makeSpecFromName(partitionNames.get(0)).keySet()); - ImmutableMap.Builder partitions = ImmutableMap.builder(); - client.getClient().set_ugi(user, ImmutableList.of()); - for (Partition partition : client.getPartitionsByNames(databaseName, tableName, partitionNames)) { - String partitionId = FileUtils.makePartName(partitionColumnNames, partition.getValues(), null); - partitions.put(partitionId, partition); - } - return Optional.of(partitions.build()); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - // assume none of the partitions in the batch are available - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional
getTable(String user, String databaseName, String tableName) - { - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, HiveViewNotSupportedException.class) - .stopOnIllegalExceptions() - .run("getTable", stats.getGetTable().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - Table table = client.getTable(databaseName, tableName); - if (table.getTableType().equals(TableType.VIRTUAL_VIEW.name()) && (!isPrestoView(table))) { - throw new HiveViewNotSupportedException(new SchemaTableName(databaseName, tableName)); - } - return Optional.of(table); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Set getRoles(String user) - { - try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { - List roles = client.listRoles(user, USER); - if (roles == null) { - return ImmutableSet.of(); - } - return ImmutableSet.copyOf(roles.stream() - .map(Role::getRoleName) - .collect(toSet())); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Set getDatabasePrivileges(String user, String databaseName) - { - ImmutableSet.Builder privileges = ImmutableSet.builder(); - - if (isDatabaseOwner(user, databaseName)) { - privileges.add(OWNERSHIP); - } - privileges.addAll(getPrivileges(user, new HiveObjectRef(HiveObjectType.DATABASE, databaseName, null, null, null))); - - return privileges.build(); - } - - @Override - public Set getTablePrivileges(String user, String databaseName, String tableName) - { - ImmutableSet.Builder privileges = ImmutableSet.builder(); - - if (isTableOwner(user, databaseName, tableName)) { - privileges.add(OWNERSHIP); - } - privileges.addAll(getPrivileges(user, new HiveObjectRef(HiveObjectType.TABLE, databaseName, tableName, null, null))); - - return privileges.build(); - } - - private Set getPrivileges(String user, HiveObjectRef objectReference) - { - ImmutableSet.Builder privileges = ImmutableSet.builder(); - try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { - PrincipalPrivilegeSet privilegeSet = client.getPrivilegeSet(objectReference, user, null); - - if (privilegeSet != null) { - Map> userPrivileges = privilegeSet.getUserPrivileges(); - if (userPrivileges != null) { - privileges.addAll(toGrants(userPrivileges.get(user))); - } - for (List rolePrivileges : privilegeSet.getRolePrivileges().values()) { - privileges.addAll(toGrants(rolePrivileges)); - } - // We do not add the group permissions as Hive does not seem to process these - } - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - - return privileges.build(); - } - - private static Set toGrants(List userGrants) - { - if (userGrants == null) { - return ImmutableSet.of(); - } - - ImmutableSet.Builder privileges = ImmutableSet.builder(); - for (PrivilegeGrantInfo userGrant : userGrants) { - privileges.addAll(parsePrivilege(userGrant)); - if (userGrant.isGrantOption()) { - privileges.add(HivePrivilege.GRANT); - } - } - return privileges.build(); - } -} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java deleted file mode 100644 index 7778e2ba9010..000000000000 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.hive.metastore; - -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -public class UserBasedHiveMetastoreStats -{ - private final HiveMetastoreApiStats getAllDatabases = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getDatabase = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getAllTables = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getAllViews = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getTable = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getPartitionNames = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getPartitionNamesPs = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getPartitionByName = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getPartitionsByNames = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats createTable = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats dropTable = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats alterTable = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats addPartitions = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats dropPartition = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats dropPartitionByName = new HiveMetastoreApiStats(); - - @Managed - @Nested - public HiveMetastoreApiStats getGetAllDatabases() - { - return getAllDatabases; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetDatabase() - { - return getDatabase; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetAllTables() - { - return getAllTables; - } - - @Managed - @Nested - public HiveMetastoreApiStats getAllViews() - { - return getAllViews; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetTable() - { - return getTable; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetPartitionNames() - { - return getPartitionNames; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetPartitionNamesPs() - { - return getPartitionNamesPs; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetPartitionByName() - { - return getPartitionByName; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetPartitionsByNames() - { - return getPartitionsByNames; - } - - @Managed - @Nested - public HiveMetastoreApiStats getCreateTable() - { - return createTable; - } - - @Managed - @Nested - public HiveMetastoreApiStats getDropTable() - { - return dropTable; - } - - @Managed - @Nested - public HiveMetastoreApiStats getAlterTable() - { - return alterTable; - } - - @Managed - @Nested - public HiveMetastoreApiStats getAddPartitions() - { - return addPartitions; - } - - @Managed - @Nested - public HiveMetastoreApiStats getDropPartition() - { - return dropPartition; - } - - @Managed - @Nested - public HiveMetastoreApiStats getDropPartitionByName() - { - return dropPartitionByName; - } -} From fe37227c083ee62b209845b90799d4aa0ea3a238 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 23 Nov 2015 11:45:41 -0800 Subject: [PATCH 23/83] Prepare for release 0.126-tw-0.7. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 52e309f6df08..414fc84a115a 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.126 + 0.126-tw-0.7 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 0cc6cf09234b..9a133d8e8f0f 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 9e0761b15f3d..3f218464e80f 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 3866f58c52ee..96fc271fb37c 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126 + 0.126-tw-0.7 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index e91a54b509c9..5344477eda37 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 03ef5b2a0230..0a882093d15c 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 46f0657d2713..ad4ec98b26bc 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 93a19ec2c54b..ac9647ce93ab 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 55dc44d383e4..f675e09c07fa 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 52f83773c3fd..e459541f3f49 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index a5a2688113f8..3a153832ad5b 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 38793b1cc542..4b412e4133e1 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 7fcd9c617283..c1bfac684b13 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 0c6c17a88228..29994c4839ac 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 9a80e1608146..5498b04ad31e 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index d1b20af9b4b8..275b9946f55d 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 5934d498d5f8..5c7cd51fbd15 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 0ed7ee924f7d..bac1e1f514d0 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index ea9e6d448a2a..013447fe2411 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index c0c6cf3bdba7..ddaf72fbbe69 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 2d4c8893a501..45c9d8a3ee59 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 224f07d32d5b..892e52942c88 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index baa497e03aa1..a04a1810cacf 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 31111245dbea..7e2c6e0aafe6 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index ada21024306c..d1056e3df9a7 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126 + 0.126-tw-0.7 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 71f1ea8d79e3..d9a3f6ef1fbb 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 7fcfc62e80c6..2bfe3c924b40 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index e438bee2d5a7..e512dc5d32f8 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index d6dc67beb915..e053c3aac08b 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index e919eea24cdf..5832a24c82d5 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index df3aaba4e79f..5c9bf5584c84 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index b069db9c4c96..a9344873b22f 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 59fbc9d4518f..966add9b8b0a 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 5842d70b3388..016b69ffc3b1 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126 + 0.126-tw-0.7 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 8362c3721e8a..60821655b780 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index cc46b69468d1..08dafd617da6 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-verifier From 344c8f0b4e90e38a26f443b9ad090f632ae54478 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 8 Dec 2015 11:19:49 -0800 Subject: [PATCH 24/83] HiveSplitManager to start HiveSplitLoader as session user --- .../presto/hive/HiveSplitManager.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index 8f63692545bc..fcca3ab5bdc1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -34,9 +34,12 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.security.UserGroupInformation; import javax.inject.Inject; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -180,9 +183,22 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa recursiveDfsWalkerEnabled); HiveSplitSource splitSource = new HiveSplitSource(connectorId, maxOutstandingSplits, hiveSplitLoader, executor); - hiveSplitLoader.start(splitSource); - return splitSource; + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); + + try { + ugi.doAs((PrivilegedExceptionAction) + () -> { + hiveSplitLoader.start(splitSource); + return null; + } + ); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not runAs " + ugi.getUserName(), e); + } + + return splitSource; } private Iterable getPartitionMetadata(Table table, SchemaTableName tableName, List hivePartitions) From 378c4ed24ff10ca8954d1857aa1fd55d949c7873 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 8 Dec 2015 12:30:36 -0800 Subject: [PATCH 25/83] fixing indentation --- .../facebook/presto/hive/HiveSplitManager.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index fcca3ab5bdc1..b04d13ae99cf 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -187,18 +187,18 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); try { - ugi.doAs((PrivilegedExceptionAction) - () -> { - hiveSplitLoader.start(splitSource); - return null; - } - ); + ugi.doAs((PrivilegedExceptionAction) + () -> { + hiveSplitLoader.start(splitSource); + return null; + } + ); } catch (IOException | InterruptedException e) { - throw new RuntimeException("Could not runAs " + ugi.getUserName(), e); + throw new RuntimeException("Could not runAs " + ugi.getUserName(), e); } - return splitSource; + return splitSource; } private Iterable getPartitionMetadata(Table table, SchemaTableName tableName, List hivePartitions) From 484e2554ecd3f8695231f2ca624a7578c33d6686 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 8 Dec 2015 12:48:49 -0800 Subject: [PATCH 26/83] fixing indentation --- .../java/com/facebook/presto/hive/HiveSplitManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index b04d13ae99cf..aa6ba715b7f6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -188,10 +188,10 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa try { ugi.doAs((PrivilegedExceptionAction) - () -> { - hiveSplitLoader.start(splitSource); - return null; - } + () -> { + hiveSplitLoader.start(splitSource); + return null; + } ); } catch (IOException | InterruptedException e) { From 4300a90172c84b8f5cd3587d1c0a4358b5114e40 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 8 Dec 2015 21:17:06 -0800 Subject: [PATCH 27/83] Updating version from to 0.126-tw-0.8 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 414fc84a115a..87c608a57ce1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.126-tw-0.7 + 0.126-tw-0.8 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 9a133d8e8f0f..2b54b534ec61 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 3f218464e80f..24f3f314d8cf 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 96fc271fb37c..60b0e963643a 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.7 + 0.126-tw-0.8 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 5344477eda37..44d712cb2f74 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 0a882093d15c..bda12d19948e 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index ad4ec98b26bc..995e766931ac 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index ac9647ce93ab..235a29672d56 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index f675e09c07fa..f890cc393d9c 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index e459541f3f49..e3f6b094975c 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 3a153832ad5b..19a597ed590d 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 4b412e4133e1..1b00d6fbbc66 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index c1bfac684b13..e6884f7c1c4e 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 29994c4839ac..3cec99c7cbf4 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 5498b04ad31e..e837f4b0c0d6 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 275b9946f55d..38847ba75baa 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 5c7cd51fbd15..f7776398f801 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index bac1e1f514d0..a99b08424438 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 013447fe2411..195758c2b1ed 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index ddaf72fbbe69..1645784011d6 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 45c9d8a3ee59..3fdeff83cdaa 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 892e52942c88..85d19916b955 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a04a1810cacf..be183abcd5ec 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7e2c6e0aafe6..7f27364dea63 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index d1056e3df9a7..6b82404a6e8b 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.7 + 0.126-tw-0.8 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index d9a3f6ef1fbb..e7fbecebc5bc 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 2bfe3c924b40..a47ea27e5f17 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index e512dc5d32f8..0493d38e3e4b 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index e053c3aac08b..67c18607589f 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 5832a24c82d5..cd423b82ff29 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 5c9bf5584c84..93a4a5070b16 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index a9344873b22f..b7542fc5167d 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 966add9b8b0a..969316f4e454 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 016b69ffc3b1..d1f9516597a3 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.7 + 0.126-tw-0.8 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 60821655b780..4bfbd1272ef5 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 08dafd617da6..80e27a5c2df0 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-verifier From b00c53a3a350b295bdddaaabbaf4dd9806045cbf Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 16 Dec 2015 21:12:53 -0800 Subject: [PATCH 28/83] Revert runAsUser in preparation for 0.130 upgrade --- .../facebook/presto/hive/HiveSplitManager.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index aa6ba715b7f6..8f63692545bc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -34,12 +34,9 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.security.UserGroupInformation; import javax.inject.Inject; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -183,20 +180,7 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa recursiveDfsWalkerEnabled); HiveSplitSource splitSource = new HiveSplitSource(connectorId, maxOutstandingSplits, hiveSplitLoader, executor); - - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); - - try { - ugi.doAs((PrivilegedExceptionAction) - () -> { - hiveSplitLoader.start(splitSource); - return null; - } - ); - } - catch (IOException | InterruptedException e) { - throw new RuntimeException("Could not runAs " + ugi.getUserName(), e); - } + hiveSplitLoader.start(splitSource); return splitSource; } From adb149c1a125ef6ddc7c97dc24084727b2396766 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 16 Dec 2015 21:29:15 -0800 Subject: [PATCH 29/83] Changing pom version to 0.130 to avoid merge conflicts when merging in 0.130 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 87c608a57ce1..7b7b6132193d 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.126-tw-0.8 + 0.130 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 2b54b534ec61..2238de516139 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 24f3f314d8cf..c9055bbaf619 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 60b0e963643a..1503671c800f 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.8 + 0.130 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 44d712cb2f74..a4db98340a48 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index bda12d19948e..e91e1350506c 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 995e766931ac..b1478999daae 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 235a29672d56..68904e2d5b5f 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index f890cc393d9c..a3fea8108719 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index e3f6b094975c..3c9c1d32c05c 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 19a597ed590d..fd26c464678e 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 1b00d6fbbc66..1ba9d816d070 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index e6884f7c1c4e..146eee7554fc 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 3cec99c7cbf4..476ff742e5fe 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index e837f4b0c0d6..c36826cb6065 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 38847ba75baa..511b51e67e91 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index f7776398f801..5781bb8854fe 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index a99b08424438..88092f60e0e7 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 195758c2b1ed..8c3f7eaa4550 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 1645784011d6..46f9c3b44552 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 3fdeff83cdaa..1f284d530c36 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 85d19916b955..e59590ac04ef 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index be183abcd5ec..7ebf10590ad7 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7f27364dea63..23733201a738 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 6b82404a6e8b..d4bcbca7ca86 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.8 + 0.130 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index e7fbecebc5bc..dfabac002bf5 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index a47ea27e5f17..f8e944107093 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 0493d38e3e4b..91b8272d48a6 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 67c18607589f..8747490178eb 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index cd423b82ff29..d2cf4984cc61 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 93a4a5070b16..aaf670e2913d 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index b7542fc5167d..c11cac6fa8a6 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 969316f4e454..986d71502f89 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index d1f9516597a3..624160ecc0c8 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.8 + 0.130 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 4bfbd1272ef5..3be286cebe3a 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 80e27a5c2df0..6963319d3361 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-verifier From e84c4a4e50a5dd1555c4dcdde2531313429fe713 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 16 Dec 2015 21:36:07 -0800 Subject: [PATCH 30/83] Prepare for release 0.130-tw-0.9 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 53aa0c6deed1..c627fc780d5a 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.130 + 0.130-tw-0.9 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 2238de516139..b1cdc92b7133 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index c9055bbaf619..2c1cf1e8232d 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 1503671c800f..71ac5b6b6a71 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130 + 0.130-tw-0.9 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index a4db98340a48..7edb8a4b301f 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index e91e1350506c..70adcfa34e45 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index b1478999daae..9dcd4c01479e 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 68904e2d5b5f..7cee57559957 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index a3fea8108719..0a2a80999767 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 3c9c1d32c05c..c794568a3064 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index fd26c464678e..31a0e5011eba 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 1ba9d816d070..46480d8d82e9 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 146eee7554fc..64deba3e5bc7 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 476ff742e5fe..134384e326a8 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 71d9f68ff8b8..8bcbb3075b9d 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index d2555a1ddba6..c7d937faa290 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 5781bb8854fe..baa24b1ef8de 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 88092f60e0e7..ff01a6845134 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 8c3f7eaa4550..bcf29c7aa29a 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 46f9c3b44552..4b195e0d7925 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 1f284d530c36..d8639669a4bc 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 223ecf28ad9f..073f5f3cafa3 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 7ebf10590ad7..bbfefe90b54c 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 23733201a738..8801d9be3117 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index d4bcbca7ca86..7ff1c69c918d 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130 + 0.130-tw-0.9 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 0cf57ffa36e9..b95b1d27d1fb 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index f8e944107093..4d22fb8ad760 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 91b8272d48a6..4f95e5638e5d 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 8747490178eb..db15e14079e3 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index d2cf4984cc61..8c6a691ba085 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 81ceb2227166..618475375aaf 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index c11cac6fa8a6..cecc9e929c4f 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 986d71502f89..e3aa177c836c 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 624160ecc0c8..07ecc82a8563 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130 + 0.130-tw-0.9 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 3be286cebe3a..2ce2e06261ff 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 9ffdd566bcd6..5464d09e2217 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-verifier From 3861dc30f59e8db824d84931694ce413564b5cb4 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 16 Dec 2015 23:41:42 -0800 Subject: [PATCH 31/83] Remove empty line to fix stylecheck error --- .../com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java index 9ccf999edac4..6b82efa22f97 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java @@ -77,7 +77,6 @@ public void destroy() zkMetastoreMonitor.close(); zkClient.close(); zkServer.close(); - } @BeforeTest From 7dd60e54e7d853ea16e3d7473b2af0097037349c Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 22 Dec 2015 17:10:01 -0800 Subject: [PATCH 32/83] Allow Hive queries to read as the querying user --- .../hive/BackgroundHiveSplitLoader.java | 24 +++++++++++++++++++ .../presto/hive/HiveClientConfig.java | 15 ++++++++++++ .../presto/hive/HiveSessionProperties.java | 11 +++++++++ 3 files changed, 50 insertions(+) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 0c854f66abcd..d15374abcf75 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -38,8 +38,10 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collections; import java.util.Deque; @@ -154,6 +156,28 @@ private class HiveSplitLoaderTask { @Override public TaskStatus process() + { + if (HiveSessionProperties.getReadAsQueryUser(session)) { + try { + // TODO: Configure hadoop to allow sqlsystem to impersonate all presto users and + // change to the approach below. + // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html + // UserGroupInformation ugi = UserGroupInformation.createProxyUser( + // session.getUser(), UserGroupInformation.getLoginUser()); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); + + return ugi.doAs((PrivilegedExceptionAction) this::doProcess); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not runAs " + session.getUser(), e); + } + } + else { + return doProcess(); + } + } + + private TaskStatus doProcess() { while (true) { if (stopped) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java index 21cf49ce0b87..8ddca5cd58ba 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java @@ -67,6 +67,8 @@ public class HiveClientConfig private boolean allowCorruptWritesForTesting; + private boolean readAsQueryUser = false; + private Duration metastoreCacheTtl = new Duration(1, TimeUnit.HOURS); private Duration metastoreRefreshInterval = new Duration(1, TimeUnit.SECONDS); private int maxMetastoreRefreshThreads = 100; @@ -291,6 +293,19 @@ public HiveClientConfig setAllowCorruptWritesForTesting(boolean allowCorruptWrit return this; } + public boolean getReadAsQueryUser() + { + return readAsQueryUser; + } + + @Config("hive.read-as-query-user") + @ConfigDescription("When querying hive read data as the user submitting the query instead of as the presto daemon user") + public HiveClientConfig setReadAsQueryUser(boolean readAsQueryUser) + { + this.readAsQueryUser = readAsQueryUser; + return this; + } + public boolean getAllowAddColumn() { return this.allowAddColumn; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java index 37bf715c825f..6dd1bc9852a0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java @@ -28,6 +28,7 @@ public final class HiveSessionProperties { private static final String FORCE_LOCAL_SCHEDULING = "force_local_scheduling"; + private static final String READ_AS_QUERY_USER = "read_as_query_user"; private static final String OPTIMIZED_READER_ENABLED = "optimized_reader_enabled"; private static final String ORC_MAX_MERGE_DISTANCE = "orc_max_merge_distance"; private static final String ORC_MAX_BUFFER_SIZE = "orc_max_buffer_size"; @@ -51,6 +52,11 @@ public HiveSessionProperties(HiveClientConfig config) "Enable optimized readers", config.isOptimizedReaderEnabled(), true), + booleanSessionProperty( + READ_AS_QUERY_USER, + "Query reads happen as the user submitting the query", + config.getReadAsQueryUser(), + true), dataSizeSessionProperty( ORC_MAX_MERGE_DISTANCE, "ORC: Maximum size of gap between two reads to merge into a single read", @@ -93,6 +99,11 @@ public static boolean isOptimizedReaderEnabled(ConnectorSession session) return session.getProperty(OPTIMIZED_READER_ENABLED, Boolean.class); } + public static boolean getReadAsQueryUser(ConnectorSession session) + { + return session.getProperty(READ_AS_QUERY_USER, Boolean.class); + } + public static boolean isParquetOptimizedReaderEnabled(ConnectorSession session) { return session.getProperty(PARQUET_OPTIMIZED_READER_ENABLED, Boolean.class); From efc25af93ecf4c79f60aa874dc20c8781e735f1e Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 23 Dec 2015 09:45:28 -0800 Subject: [PATCH 33/83] add links to jiras --- .../com/facebook/presto/hive/BackgroundHiveSplitLoader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index d15374abcf75..1123e33a1f42 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -159,8 +159,8 @@ public TaskStatus process() { if (HiveSessionProperties.getReadAsQueryUser(session)) { try { - // TODO: Configure hadoop to allow sqlsystem to impersonate all presto users and - // change to the approach below. + // TODO: Configure hadoop to allow sqlsystem to impersonate all presto users (HADOOPINFRA-7081) + // and then change to the approach below (IQ-85). // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html // UserGroupInformation ugi = UserGroupInformation.createProxyUser( // session.getUser(), UserGroupInformation.getLoginUser()); From ef06e47416518d7478ab721e01cc5a59cbeb0f81 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 23 Dec 2015 15:16:45 -0800 Subject: [PATCH 34/83] Clean up comments and constant order --- .../presto/hive/BackgroundHiveSplitLoader.java | 4 ++-- .../facebook/presto/hive/HiveSessionProperties.java | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 1123e33a1f42..82b6d7cb54da 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -159,8 +159,8 @@ public TaskStatus process() { if (HiveSessionProperties.getReadAsQueryUser(session)) { try { - // TODO: Configure hadoop to allow sqlsystem to impersonate all presto users (HADOOPINFRA-7081) - // and then change to the approach below (IQ-85). + // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users + // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html // UserGroupInformation ugi = UserGroupInformation.createProxyUser( // session.getUser(), UserGroupInformation.getLoginUser()); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java index 6dd1bc9852a0..ab1d4d2c59bf 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java @@ -28,13 +28,13 @@ public final class HiveSessionProperties { private static final String FORCE_LOCAL_SCHEDULING = "force_local_scheduling"; - private static final String READ_AS_QUERY_USER = "read_as_query_user"; private static final String OPTIMIZED_READER_ENABLED = "optimized_reader_enabled"; private static final String ORC_MAX_MERGE_DISTANCE = "orc_max_merge_distance"; private static final String ORC_MAX_BUFFER_SIZE = "orc_max_buffer_size"; private static final String ORC_STREAM_BUFFER_SIZE = "orc_stream_buffer_size"; private static final String PARQUET_PREDICATE_PUSHDOWN_ENABLED = "parquet_predicate_pushdown_enabled"; private static final String PARQUET_OPTIMIZED_READER_ENABLED = "parquet_optimized_reader_enabled"; + private static final String READ_AS_QUERY_USER = "read_as_query_user"; private final List> sessionProperties; @@ -99,11 +99,6 @@ public static boolean isOptimizedReaderEnabled(ConnectorSession session) return session.getProperty(OPTIMIZED_READER_ENABLED, Boolean.class); } - public static boolean getReadAsQueryUser(ConnectorSession session) - { - return session.getProperty(READ_AS_QUERY_USER, Boolean.class); - } - public static boolean isParquetOptimizedReaderEnabled(ConnectorSession session) { return session.getProperty(PARQUET_OPTIMIZED_READER_ENABLED, Boolean.class); @@ -129,6 +124,11 @@ public static boolean isParquetPredicatePushdownEnabled(ConnectorSession session return session.getProperty(PARQUET_PREDICATE_PUSHDOWN_ENABLED, Boolean.class); } + public static boolean getReadAsQueryUser(ConnectorSession session) + { + return session.getProperty(READ_AS_QUERY_USER, Boolean.class); + } + public static PropertyMetadata dataSizeSessionProperty(String name, String description, DataSize defaultValue, boolean hidden) { return new PropertyMetadata<>( From 2200cdef2bbed2e6260dfd2d2abdaccc9bab3f80 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Mon, 4 Jan 2016 10:52:56 -0800 Subject: [PATCH 35/83] Upgrading to 0.130-tw-0.10 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index c627fc780d5a..45f1aab240a9 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.130-tw-0.9 + 0.130-tw-0.10 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index b1cdc92b7133..254d1a0a97df 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 2c1cf1e8232d..3551e5b9ba79 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 71ac5b6b6a71..758f46b6829f 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.9 + 0.130-tw-0.10 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 7edb8a4b301f..3098d1160fb0 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 70adcfa34e45..1e6d4890fab2 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 9dcd4c01479e..e7cb584603c0 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 7cee57559957..25be93acdfe5 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0a2a80999767..5cd6892eb6ed 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index c794568a3064..32c2f9e32945 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 31a0e5011eba..593000f9eb64 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 46480d8d82e9..938bb689ee59 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 64deba3e5bc7..06b43819b485 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 134384e326a8..a23dc2923fe8 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 8bcbb3075b9d..0fc8dfc22def 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index c7d937faa290..fe3d827a1628 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index baa24b1ef8de..cfeeb3122bda 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index ff01a6845134..afe37061a74c 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index bcf29c7aa29a..19aa2a3e7f7b 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 4b195e0d7925..4e15544c3e32 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index d8639669a4bc..eb7bb0041273 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 073f5f3cafa3..58ed96057649 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index bbfefe90b54c..659f51f9e7ee 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 8801d9be3117..8b73a4081cee 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 7ff1c69c918d..1868af2c227c 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.9 + 0.130-tw-0.10 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index b95b1d27d1fb..47a876a3f1a1 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 4d22fb8ad760..bedfad6e0542 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 4f95e5638e5d..67ac2b629ac3 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index db15e14079e3..d9e657b67322 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 8c6a691ba085..1369629d997b 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 618475375aaf..be03619e8409 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index cecc9e929c4f..300a83451b02 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index e3aa177c836c..b826dfab48a9 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 07ecc82a8563..0efab43b7181 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.9 + 0.130-tw-0.10 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 2ce2e06261ff..34b329a0af18 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 5464d09e2217..a34d0649d123 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-verifier From 1b38bdf471eb43e90127200d828f8d2ea02b3101 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Mon, 4 Jan 2016 11:46:32 -0800 Subject: [PATCH 36/83] Updating unit tests for ReadAsQueryUser flag --- .../java/com/facebook/presto/hive/TestHiveClientConfig.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java index 08311e3c09e1..b7647ee99677 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java @@ -66,6 +66,7 @@ public void testDefaults() .setResourceConfigFiles((String) null) .setHiveStorageFormat(HiveStorageFormat.RCBINARY) .setRespectTableFormat(true) + .setReadAsQueryUser(false) .setImmutablePartitions(false) .setMaxPartitionsPerWriter(100) .setUseParquetColumnNames(false) @@ -132,6 +133,7 @@ public void testExplicitPropertyMappings() .put("hive.max-concurrent-file-renames", "100") .put("hive.assume-canonical-partition-keys", "true") .put("hive.parquet.use-column-names", "true") + .put("hive.read-as-query-user", "true") .put("hive.s3.aws-access-key", "abc123") .put("hive.s3.aws-secret-key", "secret") .put("hive.s3.use-instance-credentials", "false") @@ -185,6 +187,7 @@ public void testExplicitPropertyMappings() .setVerifyChecksum(false) .setResourceConfigFiles(ImmutableList.of("/foo.xml", "/bar.xml")) .setHiveStorageFormat(HiveStorageFormat.SEQUENCEFILE) + .setReadAsQueryUser(true) .setRespectTableFormat(false) .setImmutablePartitions(true) .setMaxPartitionsPerWriter(222) From 9f646d1af11a8c43e350d4d4fbc4a9ca31d83019 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Wed, 6 Jan 2016 12:33:56 -0800 Subject: [PATCH 37/83] Avoid NPE when ZK changed state is related to connection change. --- .../presto/hive/ZookeeperMetastoreMonitor.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java index 002eefe5cc13..3789ca3c3271 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java @@ -94,31 +94,25 @@ private HostAndPort deserialize(byte[] bytes) @Override public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { - String node = ZKPaths.getNodeFromPath(event.getData().getPath()); - switch (event.getType()) { - case CHILD_ADDED: { - HostAndPort hostPort = deserialize(event.getData().getData()); - log.info("child added: " + node + ": " + hostPort); - servers.put(node, hostPort); - break; - } - + case CHILD_ADDED: case CHILD_UPDATED: { HostAndPort hostPort = deserialize(event.getData().getData()); + String node = ZKPaths.getNodeFromPath(event.getData().getPath()); log.info("child updated: " + node + ": " + hostPort); servers.put(node, hostPort); break; } case CHILD_REMOVED: { + String node = ZKPaths.getNodeFromPath(event.getData().getPath()); log.info("child removed: " + node); servers.remove(node); break; } default: - log.info("connection state changed: " + node); + log.info("connection state changed: " + event.getType()); break; } } From 48973308e108c13354c5c7b33823584fd32cad90 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 12 Jan 2016 10:38:13 -0800 Subject: [PATCH 38/83] Cache the UGI object per user in BackgroundHiveSplitLoader --- .../hive/BackgroundHiveSplitLoader.java | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 82b6d7cb54da..4c3c6e2e1312 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -51,6 +51,7 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; @@ -72,6 +73,11 @@ public class BackgroundHiveSplitLoader { public static final CompletableFuture COMPLETED_FUTURE = CompletableFuture.completedFuture(null); + // Every instance of a UserGroupInformation object for a given user has a unique hashcode, due + // to the hashCode() impl. If we don't cache the UGI per-user here, there will be a memory leak + // in the PrestoFileSystemCache. + private static final Map UGI_CACHE = new ConcurrentHashMap<>(); + private final String connectorId; private final Table table; private final Optional bucket; @@ -140,8 +146,26 @@ public BackgroundHiveSplitLoader( public void start(HiveSplitSource splitSource) { this.hiveSplitSource = splitSource; + + UserGroupInformation ugi = null; + + if (HiveSessionProperties.getReadAsQueryUser(session)) { + String user = session.getUser(); + ugi = UGI_CACHE.get(user); + + if (ugi == null) { + // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users + // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). + // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html + // UserGroupInformation ugi = UserGroupInformation.createProxyUser( + // session.getUser(), UserGroupInformation.getLoginUser()); + ugi = UserGroupInformation.createRemoteUser(user); + UGI_CACHE.put(user, ugi); + } + } + for (int i = 0; i < maxPartitionBatchSize; i++) { - ResumableTasks.submit(executor, new HiveSplitLoaderTask()); + ResumableTasks.submit(executor, new HiveSplitLoaderTask(ugi)); } } @@ -154,18 +178,18 @@ public void stop() private class HiveSplitLoaderTask implements ResumableTask { + private UserGroupInformation ugi; + + public HiveSplitLoaderTask(UserGroupInformation ugi) + { + this.ugi = ugi; + } + @Override public TaskStatus process() { - if (HiveSessionProperties.getReadAsQueryUser(session)) { + if (ugi != null) { try { - // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users - // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). - // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html - // UserGroupInformation ugi = UserGroupInformation.createProxyUser( - // session.getUser(), UserGroupInformation.getLoginUser()); - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); - return ugi.doAs((PrivilegedExceptionAction) this::doProcess); } catch (IOException | InterruptedException e) { From 53ae65b2def1cf7c7dc64e2d46037d1869db5453 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 12 Jan 2016 11:44:36 -0800 Subject: [PATCH 39/83] Upgrading version to 0.130-tw-0.11 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 45f1aab240a9..18a82637339e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.130-tw-0.10 + 0.130-tw-0.11 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 254d1a0a97df..38f39e5bed6e 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 3551e5b9ba79..c0681a377e35 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 758f46b6829f..ad5f16c3d1ce 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.10 + 0.130-tw-0.11 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 3098d1160fb0..9d9927732b12 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 1e6d4890fab2..57ede93b43d9 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index e7cb584603c0..d7332844143c 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 25be93acdfe5..e9448441e73f 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 5cd6892eb6ed..ac21d14d80ec 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 32c2f9e32945..854bb94bd03a 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 593000f9eb64..0c1b2d9b6de5 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 938bb689ee59..0425c49408ed 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 06b43819b485..bfcc07f1f1e8 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index a23dc2923fe8..e82ef33a456f 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 0fc8dfc22def..f5e8c4c1953f 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index fe3d827a1628..3602a338b4f5 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index cfeeb3122bda..897fbb6cad37 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index afe37061a74c..82f8ab1f75e9 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 19aa2a3e7f7b..b88f23e6c6a6 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 4e15544c3e32..9531e434a54a 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index eb7bb0041273..dd68037b9691 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 58ed96057649..3f4f329dc9e9 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 659f51f9e7ee..20cf118a617c 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 8b73a4081cee..81edf97a3c5d 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 1868af2c227c..db8f461db9f6 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.10 + 0.130-tw-0.11 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 47a876a3f1a1..879bbc03bcf0 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index bedfad6e0542..224ea2174faf 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 67ac2b629ac3..d22265c2e95c 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index d9e657b67322..562371fbc934 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 1369629d997b..399083032f49 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index be03619e8409..64f4a5aae122 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 300a83451b02..a1b8f7b4b9f7 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index b826dfab48a9..0aa645c9613e 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 0efab43b7181..7332273d684a 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.10 + 0.130-tw-0.11 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 34b329a0af18..bef17e132820 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index a34d0649d123..e894a2a8420b 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-verifier From eff466b617cc1d1dbf1f25b805b2ecdde33c184c Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Wed, 13 Jan 2016 10:56:49 -0800 Subject: [PATCH 40/83] Change version to 0.132. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 18a82637339e..0505afdef136 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.130-tw-0.11 + 0.132 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 38f39e5bed6e..0757573fdeac 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index c0681a377e35..7cfc90dd1d09 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index ad5f16c3d1ce..4e97c070cf5b 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.11 + 0.132 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 9d9927732b12..1288733d1437 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 57ede93b43d9..3dca182b2de4 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index d7332844143c..135f8845791f 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index e9448441e73f..e6b2500dbed6 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index ac21d14d80ec..892071a00abe 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 854bb94bd03a..1f88313cc127 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 0c1b2d9b6de5..ad4b4a2a0de7 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 0425c49408ed..add0c37510b7 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index bfcc07f1f1e8..8f3ed0661a7a 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index e82ef33a456f..753f94c30bdf 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f5e8c4c1953f..a05910678f98 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 3602a338b4f5..9306604a38b3 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 897fbb6cad37..57c23c70c40a 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 82f8ab1f75e9..311ff49ce349 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index b88f23e6c6a6..b0438d65c2ae 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 9531e434a54a..d3f04d7dd2aa 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index dd68037b9691..c0a1a1a32cae 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 3f4f329dc9e9..0b88fc338d2b 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 20cf118a617c..0d05fa046030 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 81edf97a3c5d..a4009d5b00ad 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index db8f461db9f6..4423dbd7e9e0 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.11 + 0.132 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 879bbc03bcf0..3b69b26d948f 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 224ea2174faf..ea37942e5e63 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index d22265c2e95c..498a260bd5d2 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 562371fbc934..2ff6d2c1a64e 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 399083032f49..9d135250631d 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 64f4a5aae122..b277f10bf706 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index a1b8f7b4b9f7..36717f601a71 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 0aa645c9613e..27a65f8c462b 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 7332273d684a..23b0093b7898 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.11 + 0.132 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index bef17e132820..bcc4768dedf1 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index e894a2a8420b..7cfaddeadd23 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-verifier From 14fdca19c6357d2cae3bf743f6f5cce69dee38e8 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Wed, 13 Jan 2016 11:04:41 -0800 Subject: [PATCH 41/83] Change version to 0.132-tw-0.12. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 09213bb52fad..0a7310f24e82 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132 + 0.132-tw-0.12 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 0757573fdeac..1baffbd7964d 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 7cfc90dd1d09..c5d1e97aed38 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 4e97c070cf5b..720c269b8da6 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132 + 0.132-tw-0.12 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index e51c2780dd01..8e9f44465cd2 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index bbf43c188d3a..a1ca82981c41 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 3dca182b2de4..228bef817de6 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 135f8845791f..f17279106bfa 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index e6b2500dbed6..80c85709d67f 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 892071a00abe..3201c365c7eb 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 1f88313cc127..07bb8a3fedfd 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index ad4b4a2a0de7..d2c77960ab34 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index add0c37510b7..e5c4dbf0ded8 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 8f3ed0661a7a..9b50071e2147 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 753f94c30bdf..c33ce09bbf18 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index a05910678f98..062cd3b3acc7 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9306604a38b3..9c99be4db232 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 57c23c70c40a..7d1d123ba81d 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 311ff49ce349..5ab9e91e369c 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index e5c954b4aea4..ba634e0b5580 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index d3f04d7dd2aa..f41514dd6603 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index c0a1a1a32cae..dc92ae9833ce 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 0b88fc338d2b..09c469a95f67 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 0d05fa046030..4842899667bb 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index a4009d5b00ad..345c15453b13 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 4423dbd7e9e0..a9844a05f653 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132 + 0.132-tw-0.12 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 3b69b26d948f..d9b862a8316a 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index ea37942e5e63..ebfd82dfe86f 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 498a260bd5d2..3098dbba128e 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 2ff6d2c1a64e..c8bad3c5fedd 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 9d135250631d..c3e87fb73c03 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index b277f10bf706..80d537ff1e7f 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 36717f601a71..7021f5b56d1b 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 27a65f8c462b..2169be00f880 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 8113b83bbc4e..0e30a1f586fb 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132 + 0.132-tw-0.12 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index bcc4768dedf1..8a103229074e 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 7cfaddeadd23..18c5b2961b4a 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-verifier From 310441d3775ddacce9c69c98c602e75262822718 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 14 Jan 2016 23:31:43 -0800 Subject: [PATCH 42/83] HivePageSourceProvider to support query as user from workers --- .../hive/BackgroundHiveSplitLoader.java | 20 +------- .../presto/hive/HivePageSourceProvider.java | 22 +++++++++ .../facebook/presto/hive/util/UgiUtils.java | 49 +++++++++++++++++++ 3 files changed, 73 insertions(+), 18 deletions(-) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 4c3c6e2e1312..676b856e6f7c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -16,6 +16,7 @@ import com.facebook.presto.hive.util.HiveFileIterator; import com.facebook.presto.hive.util.ResumableTask; import com.facebook.presto.hive.util.ResumableTasks; +import com.facebook.presto.hive.util.UgiUtils; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.predicate.TupleDomain; @@ -51,7 +52,6 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; @@ -73,11 +73,6 @@ public class BackgroundHiveSplitLoader { public static final CompletableFuture COMPLETED_FUTURE = CompletableFuture.completedFuture(null); - // Every instance of a UserGroupInformation object for a given user has a unique hashcode, due - // to the hashCode() impl. If we don't cache the UGI per-user here, there will be a memory leak - // in the PrestoFileSystemCache. - private static final Map UGI_CACHE = new ConcurrentHashMap<>(); - private final String connectorId; private final Table table; private final Optional bucket; @@ -150,18 +145,7 @@ public void start(HiveSplitSource splitSource) UserGroupInformation ugi = null; if (HiveSessionProperties.getReadAsQueryUser(session)) { - String user = session.getUser(); - ugi = UGI_CACHE.get(user); - - if (ugi == null) { - // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users - // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). - // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html - // UserGroupInformation ugi = UserGroupInformation.createProxyUser( - // session.getUser(), UserGroupInformation.getLoginUser()); - ugi = UserGroupInformation.createRemoteUser(user); - UGI_CACHE.put(user, ugi); - } + ugi = UgiUtils.getUgi(session.getUser()); } for (int i = 0; i < maxPartitionBatchSize; i++) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java index 73abe1477345..ff1c81f19367 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.util.UgiUtils; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorPageSourceProvider; @@ -26,10 +27,13 @@ import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; import org.joda.time.DateTimeZone; import javax.inject.Inject; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -67,6 +71,24 @@ public HivePageSourceProvider( @Override public ConnectorPageSource createPageSource(ConnectorSession session, ConnectorSplit split, List columns) + { + if (HiveSessionProperties.getReadAsQueryUser(session)) { + UserGroupInformation ugi = UgiUtils.getUgi(session.getUser()); + try { + return ugi.doAs((PrivilegedExceptionAction) () -> + doCreatePageSource(session, split, columns) + ); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not runAs " + session.getUser(), e); + } + } + else { + return doCreatePageSource(session, split, columns); + } + } + + private ConnectorPageSource doCreatePageSource(ConnectorSession session, ConnectorSplit split, List columns) { HiveSplit hiveSplit = checkType(split, HiveSplit.class, "split"); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java new file mode 100644 index 000000000000..5c8e99e75efa --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.util; + +import org.apache.hadoop.security.UserGroupInformation; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Utility class to handle creating and caching the UserGroupInformation object. + */ +public class UgiUtils +{ + private UgiUtils() {} + + // Every instance of a UserGroupInformation object for a given user has a unique hashcode, due + // to the hashCode() impl. If we don't cache the UGI per-user here, there will be a memory leak + // in the PrestoFileSystemCache. + private static final Map UGI_CACHE = new ConcurrentHashMap<>(); + + public static UserGroupInformation getUgi(String user) + { + UserGroupInformation ugi = UGI_CACHE.get(user); + + if (ugi == null) { + // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users + // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). + // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html + // UserGroupInformation ugi = UserGroupInformation.createProxyUser( + // session.getUser(), UserGroupInformation.getLoginUser()); + ugi = UserGroupInformation.createRemoteUser(user); + UGI_CACHE.put(user, ugi); + } + + return ugi; + } +} From cab14e9c61d7c756810d7612e559e9bc0fafbc64 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Fri, 15 Jan 2016 13:21:38 -0800 Subject: [PATCH 43/83] Upgrading presto to 0.132-tw-13 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 0a7310f24e82..2ce743f8efef 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.12 + 0.132-tw-0.13 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 1baffbd7964d..2a013d29b68b 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index c5d1e97aed38..1f9ec5acebed 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 720c269b8da6..be910a6326bd 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.12 + 0.132-tw-0.13 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8e9f44465cd2..40e60d9700dc 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index a1ca82981c41..c9cb7dcf52a9 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 228bef817de6..49eb6dbe2b24 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index f17279106bfa..49626b15d567 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 80c85709d67f..82197cc42286 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 3201c365c7eb..93686b08e969 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 07bb8a3fedfd..e0149f18d330 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index d2c77960ab34..ae357e508d3b 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index e5c4dbf0ded8..11449f92c28c 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 9b50071e2147..33937c62d142 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index c33ce09bbf18..345bfa73fded 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 062cd3b3acc7..ccc723f4dac2 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9c99be4db232..9f34b46f98e2 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 7d1d123ba81d..28f602bc8314 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 5ab9e91e369c..9da3ebf652eb 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index ba634e0b5580..e2da193b04d9 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index f41514dd6603..1d893ba7d3e4 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index dc92ae9833ce..93f1cd722e4c 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 09c469a95f67..79af769f34dd 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 4842899667bb..58bd4e5c904a 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 345c15453b13..443b69f5a434 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index a9844a05f653..47c86b3a1ba4 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.12 + 0.132-tw-0.13 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index d9b862a8316a..0d10ea4c729b 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index ebfd82dfe86f..02d01b2c3d3c 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 3098dbba128e..c945bda7597c 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index c8bad3c5fedd..33b551a1306f 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index c3e87fb73c03..a01062233738 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 80d537ff1e7f..73363ed81103 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 7021f5b56d1b..175862f3fc8c 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 2169be00f880..8a36e2d5602f 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 0e30a1f586fb..dab3686e6aa7 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.12 + 0.132-tw-0.13 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 8a103229074e..8158dc7e5c8e 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 18c5b2961b4a..7ddccd9f434d 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-verifier From cfc21131975bdfe0756aa0bf3f4d42ba820e033f Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Sun, 31 Jan 2016 04:48:12 -0800 Subject: [PATCH 44/83] Revert PR#3, case insensitive types, since it is now supported. --- .../java/com/facebook/presto/metadata/FunctionRegistry.java | 2 +- .../main/java/com/facebook/presto/type/RowParametricType.java | 2 +- .../main/java/com/facebook/presto/spi/type/TypeSignature.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java index 8b30e3f8b760..4ebac76ce3af 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java @@ -508,7 +508,7 @@ private SqlFunction getRowFieldReference(String field, TypeSignature rowTypeSign checkState(rowType.getTypeSignature().getBase().equals(StandardTypes.ROW), "rowType is not a ROW type"); SqlFunction match = null; for (SqlFunction function : RowParametricType.ROW.createFunctions(rowType)) { - if (!function.getSignature().getName().equalsIgnoreCase(field)) { + if (!function.getSignature().getName().equals(field)) { continue; } checkArgument(match == null, "Ambiguous field %s in type %s", field, rowType.getDisplayName()); diff --git a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java index bc08fd1c3045..4705d16f87a9 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java +++ b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java @@ -68,7 +68,7 @@ public List createFunctions(Type type) RowField field = fields.get(i); int index = i; field.getName() - .ifPresent(name -> builder.add(new RowFieldReference(rowType, field.getType(), index, field.getName().get().toLowerCase()))); + .ifPresent(name -> builder.add(new RowFieldReference(rowType, field.getType(), index, field.getName().get()))); } return builder.build(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java index 75ffe4906f81..8e5d924fe399 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java @@ -189,7 +189,7 @@ private static Object parseLiteral(String literal) { if (literal.startsWith("'") || literal.endsWith("'")) { checkArgument(literal.startsWith("'") && literal.endsWith("'"), "Bad literal: '%s'", literal); - return literal.substring(1, literal.length() - 1).toLowerCase(); + return literal.substring(1, literal.length() - 1); } else { return Long.parseLong(literal); From b25b9f0ba1bba64009d3c393cf35ed2c60724fa1 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 2 Feb 2016 17:58:46 -0800 Subject: [PATCH 45/83] Update to use UGI.createProxyUser instead of createRemoteUser --- .../com/facebook/presto/hive/util/UgiUtils.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java index 5c8e99e75efa..08f2dde30ff2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java @@ -15,6 +15,7 @@ import org.apache.hadoop.security.UserGroupInformation; +import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -35,12 +36,14 @@ public static UserGroupInformation getUgi(String user) UserGroupInformation ugi = UGI_CACHE.get(user); if (ugi == null) { - // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users - // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). + // Configure hadoop to allow presto daemon user to impersonate all presto users // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html - // UserGroupInformation ugi = UserGroupInformation.createProxyUser( - // session.getUser(), UserGroupInformation.getLoginUser()); - ugi = UserGroupInformation.createRemoteUser(user); + try { + ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); + } + catch (IOException e) { + throw new RuntimeException("Could not get login user from UserGroupInformation", e); + } UGI_CACHE.put(user, ugi); } From ea2c8c36829f4cf7971e7f27555c0157bc126534 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 3 Feb 2016 11:30:26 -0800 Subject: [PATCH 46/83] Upgrade to 0.132-tw-0.14 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 2ce743f8efef..2416e615dfbe 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.13 + 0.132-tw-0.14 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 2a013d29b68b..1f6bbc8b9648 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 1f9ec5acebed..114bf047c473 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index be910a6326bd..339f20cd8a98 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.13 + 0.132-tw-0.14 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 40e60d9700dc..83e0d9c4fbc3 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index c9cb7dcf52a9..8dbaae2f8ae7 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 49eb6dbe2b24..7d5c36b52a5f 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 49626b15d567..68ca8e164066 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 82197cc42286..766de9cc7394 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 93686b08e969..d6b9e4b865f0 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index e0149f18d330..4383a60d3042 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index ae357e508d3b..2bd05749d80b 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 11449f92c28c..a795a1c8141d 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 33937c62d142..0339fdc41b7c 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 345bfa73fded..861da65b5d72 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index ccc723f4dac2..83d79d919e62 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9f34b46f98e2..97675a00841a 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 28f602bc8314..af34740a897f 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 9da3ebf652eb..c5511c6b3d64 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index e2da193b04d9..9fa2fef79ff5 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 1d893ba7d3e4..48948e33ab0b 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 93f1cd722e4c..94fff311cc77 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 79af769f34dd..5749e3cfca24 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 58bd4e5c904a..a84aeb8f9a8c 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 443b69f5a434..c15dd15cd386 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 47c86b3a1ba4..3f982d5f5342 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.13 + 0.132-tw-0.14 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 0d10ea4c729b..5630396e0e01 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 02d01b2c3d3c..4689f17b5d6c 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index c945bda7597c..47ee9d810e78 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 33b551a1306f..92a6f8c8ce62 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index a01062233738..3d7badbc6e9b 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 73363ed81103..c5c7ee7d4f3e 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 175862f3fc8c..4b718d149db5 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 8a36e2d5602f..c5aaefea41c0 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index dab3686e6aa7..fbe89c2ee7f0 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.13 + 0.132-tw-0.14 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 8158dc7e5c8e..9f39f8e46511 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 7ddccd9f434d..d1ee25a4fac7 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-verifier From 6ac4bc4686e2474eaf7804554e50fdae91458843 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 3 Feb 2016 16:47:44 -0800 Subject: [PATCH 47/83] Whitelist zeppelin temporarily --- .../main/java/com/facebook/presto/hive/util/UgiUtils.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java index 08f2dde30ff2..d5af9fb53d4c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java @@ -39,7 +39,13 @@ public static UserGroupInformation getUgi(String user) // Configure hadoop to allow presto daemon user to impersonate all presto users // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html try { - ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); + // TODO: IQ-152 roll this back once zeppelin runsAs user + if ("zeppelin".equals(user)) { + ugi = UserGroupInformation.createRemoteUser(user); + } + else { + ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); + } } catch (IOException e) { throw new RuntimeException("Could not get login user from UserGroupInformation", e); From 6d857e619eab832f58ef868bf2aa9d414f9d42cf Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 3 Feb 2016 17:13:39 -0800 Subject: [PATCH 48/83] Upgrade to 0.132-tw-0.15 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 2416e615dfbe..f14af7bda23b 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.14 + 0.132-tw-0.15 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 1f6bbc8b9648..bf932e890e47 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 114bf047c473..b0115ab5c45a 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 339f20cd8a98..787722a101b6 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.14 + 0.132-tw-0.15 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 83e0d9c4fbc3..afa61e58fbe7 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 8dbaae2f8ae7..b0ef31fe590b 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 7d5c36b52a5f..c07316019a33 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 68ca8e164066..410fd2ea0307 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 766de9cc7394..6816ba6073ab 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index d6b9e4b865f0..3ab195df116b 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 4383a60d3042..363e6ec51409 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 2bd05749d80b..84d401d9faa7 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index a795a1c8141d..a7a279c4a475 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 0339fdc41b7c..5f0851c1f5d0 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 861da65b5d72..431cee305c72 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 83d79d919e62..610809a45d5e 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 97675a00841a..65ef3e4a3779 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index af34740a897f..39edd5685be9 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index c5511c6b3d64..2ca95314cfef 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 9fa2fef79ff5..0932dae34401 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 48948e33ab0b..4e331b4bb39a 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 94fff311cc77..ff77684f5e9d 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 5749e3cfca24..f6b84dc54dc0 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a84aeb8f9a8c..9ed3bf4c0d63 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index c15dd15cd386..331e7c2d27d1 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 3f982d5f5342..95fce0f974a5 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.14 + 0.132-tw-0.15 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 5630396e0e01..f3e78e9044d3 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 4689f17b5d6c..adf30dff5fb0 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 47ee9d810e78..73cfbf6524a5 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 92a6f8c8ce62..fee591fefc2b 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 3d7badbc6e9b..cac03417ee1b 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index c5c7ee7d4f3e..fcc758ea371f 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 4b718d149db5..49fdfa2d07c0 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index c5aaefea41c0..9ef399614934 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index fbe89c2ee7f0..c2614a98b3cb 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.14 + 0.132-tw-0.15 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 9f39f8e46511..12409b46a8d4 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index d1ee25a4fac7..b4c38be3a538 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-verifier From 207a2cca84a79e348ea43a277add1d44981e7605 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 11 Feb 2016 14:07:07 -0800 Subject: [PATCH 49/83] Rolling back Zeppelin auth hack, add some logging --- .../java/com/facebook/presto/hive/util/UgiUtils.java | 8 +------- .../java/com/facebook/presto/server/HttpRemoteTask.java | 9 +++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java index d5af9fb53d4c..08f2dde30ff2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java @@ -39,13 +39,7 @@ public static UserGroupInformation getUgi(String user) // Configure hadoop to allow presto daemon user to impersonate all presto users // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html try { - // TODO: IQ-152 roll this back once zeppelin runsAs user - if ("zeppelin".equals(user)) { - ugi = UserGroupInformation.createRemoteUser(user); - } - else { - ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); - } + ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); } catch (IOException e) { throw new RuntimeException("Could not get login user from UserGroupInformation", e); diff --git a/presto-main/src/main/java/com/facebook/presto/server/HttpRemoteTask.java b/presto-main/src/main/java/com/facebook/presto/server/HttpRemoteTask.java index c21451070dd1..529b337add8b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/HttpRemoteTask.java +++ b/presto-main/src/main/java/com/facebook/presto/server/HttpRemoteTask.java @@ -53,6 +53,7 @@ import io.airlift.http.client.HttpClient; import io.airlift.http.client.HttpStatus; import io.airlift.http.client.Request; +import io.airlift.http.client.StaticBodyGenerator; import io.airlift.http.client.StatusResponseHandler.StatusResponse; import io.airlift.json.JsonCodec; import io.airlift.log.Logger; @@ -454,6 +455,14 @@ private synchronized void scheduleUpdate() updateErrorTracker.startRequest(); + // TODO: (billg) remove this logging or contribute it upstream + if (log.isDebugEnabled()) { + String size = "unknown"; + if (request.getBodyGenerator() instanceof StaticBodyGenerator) { + size = Integer.toString(((StaticBodyGenerator) request.getBodyGenerator()).getBody().length); + } + log.debug(String.format("scheduleUpdate POST %s, bodySize=%s sourcesSize=%s", request.getUri(), size, sources.size())); + } ListenableFuture> future = httpClient.executeAsync(request, createFullJsonResponseHandler(taskInfoCodec)); currentRequest = future; currentRequestStartNanos = System.nanoTime(); From daa37735883f0d2b2220a77664f2919bf7dd4cd4 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 11 Feb 2016 15:40:29 -0800 Subject: [PATCH 50/83] Upgrading to 0.132-tw-0.16 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index f14af7bda23b..f22df98fd73e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.15 + 0.132-tw-0.16 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index bf932e890e47..632f2069eff1 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index b0115ab5c45a..cc0eee6d5965 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 787722a101b6..05bf2895362a 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.15 + 0.132-tw-0.16 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index afa61e58fbe7..0a271e785de8 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index b0ef31fe590b..6425232da79b 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index c07316019a33..8aa9693ec1da 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 410fd2ea0307..121e534b2954 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 6816ba6073ab..ece17fc3749b 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 3ab195df116b..f25c91f065bd 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 363e6ec51409..6c0df00ff71c 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 84d401d9faa7..4068bf2c3f06 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index a7a279c4a475..455d8aa2580c 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 5f0851c1f5d0..ac7c172f3d98 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 431cee305c72..fd48780e8e96 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 610809a45d5e..f9c8aeeb7330 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 65ef3e4a3779..47bd019010f6 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 39edd5685be9..c63e02329855 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 2ca95314cfef..f67ddcb8f3c9 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 0932dae34401..eb67f46ce25a 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 4e331b4bb39a..09846fc4988d 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index ff77684f5e9d..f8fc748cb600 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index f6b84dc54dc0..47c4e2b73fe3 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 9ed3bf4c0d63..2bcb0bc67c03 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 331e7c2d27d1..4ca861f83971 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 95fce0f974a5..45afe82f6e0c 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.15 + 0.132-tw-0.16 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index f3e78e9044d3..af428336232c 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index adf30dff5fb0..371a08a68b28 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 73cfbf6524a5..2d440900ff36 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index fee591fefc2b..4c77ae0aaeed 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index cac03417ee1b..09f7c9e30b0f 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index fcc758ea371f..06b939aef70d 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 49fdfa2d07c0..1d365f90c836 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 9ef399614934..50270820bc3c 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index c2614a98b3cb..d82075da681d 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.15 + 0.132-tw-0.16 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 12409b46a8d4..846d8f330385 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index b4c38be3a538..ff14bb0ff9ee 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-verifier From f4fed25634728d7a03010e5073b2614b213a8a36 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 24 Feb 2016 09:38:56 -0800 Subject: [PATCH 51/83] Change version to 0.139 to not conflict with merge --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index f22df98fd73e..afb115e56e7b 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.16 + 0.139 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 632f2069eff1..721e5197c68d 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index cc0eee6d5965..bee89f2aca0c 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 05bf2895362a..e6b9dfe148ef 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.16 + 0.139 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 0a271e785de8..24d70e0bd523 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 6425232da79b..3cbadb8cb50c 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 8aa9693ec1da..eee77f26b681 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 121e534b2954..053443888316 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index ece17fc3749b..7e609c58d169 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index f25c91f065bd..bbdb2780c86f 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 6c0df00ff71c..74cc530dc2d6 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 4068bf2c3f06..3909da01a3e5 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 455d8aa2580c..3617329133c4 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index ac7c172f3d98..b99adacef6d4 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index fd48780e8e96..5ebdcab6d24a 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f9c8aeeb7330..db7651abb449 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 47bd019010f6..77e51c1592c6 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index c63e02329855..459740752199 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index f67ddcb8f3c9..cd5b16486e33 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index eb67f46ce25a..881acee972aa 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 09846fc4988d..2691474bab1b 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index f8fc748cb600..f7f94917820f 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 47c4e2b73fe3..e52248e66964 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 2bcb0bc67c03..bf2af5acd118 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 4ca861f83971..54d95b03d34b 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 45afe82f6e0c..4c01f1eb9160 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.16 + 0.139 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index af428336232c..4248921cc55a 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 371a08a68b28..7b6d3f420fa9 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 2d440900ff36..7b545ce1b177 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 4c77ae0aaeed..2a8ef6926ffc 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 09f7c9e30b0f..7e3151746a48 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 06b939aef70d..6460358c5dba 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 1d365f90c836..0ab884da5c84 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 50270820bc3c..cc0f53d979cb 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index d82075da681d..6d2ca1720a56 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.16 + 0.139 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 846d8f330385..db45c2072959 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index ff14bb0ff9ee..adba2eee4163 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-verifier From 2c9da6093b6e72f8634b7c874b517dc6e729ad31 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 24 Feb 2016 10:12:18 -0800 Subject: [PATCH 52/83] upgrade pom version to 0.139-tw-0.17 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 67358e743bdc..632225d50fe1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.139 + 0.139-tw-0.17 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 721e5197c68d..d84fe856c920 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index bee89f2aca0c..80781a476f81 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index e6b9dfe148ef..073888296318 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139 + 0.139-tw-0.17 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 24d70e0bd523..8117fa6549ae 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 3cbadb8cb50c..5493f10852bd 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index eee77f26b681..4339db66dc64 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 053443888316..dd2800d32a73 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 4e7fc011a28b..907443ee18ec 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index bbdb2780c86f..7df4d2893179 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 74cc530dc2d6..85b2c4f1524d 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 3909da01a3e5..1c254fb609c8 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 3617329133c4..0e58e0b8a334 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index b99adacef6d4..f75f642ef484 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 5ebdcab6d24a..22e73c96b1d0 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index db7651abb449..cc32b3878947 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 77e51c1592c6..9f748e0114ee 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 803afb3bbbd0..8be6aa536b7c 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index cd5b16486e33..ef29485ce5c9 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 73bdb896bc4d..749ca56c4191 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 2691474bab1b..34b47c0df123 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index f7f94917820f..7754bb50b25c 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index e52248e66964..422be0c545d9 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index bf2af5acd118..48218aa78fb5 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 54d95b03d34b..5891c6b4c875 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 4c01f1eb9160..6c78d18b97ca 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139 + 0.139-tw-0.17 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index e1959313da8a..8504a4df3bc8 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 7b6d3f420fa9..0719269862f1 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 7b545ce1b177..e15a8e9be8c5 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 2a8ef6926ffc..23bd57d1aef9 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 7e3151746a48..eb130f83e50d 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 6460358c5dba..f23575bcf202 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 0ab884da5c84..85bc734c1bf3 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index cc0f53d979cb..97a45f3fce0f 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 6d2ca1720a56..75edb19713ae 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139 + 0.139-tw-0.17 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index db45c2072959..730294962ce5 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index ad246ed83646..507e3ebe0c51 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-verifier From 582df38cca94dbe0f447673c34973badc96397e6 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 24 Feb 2016 14:48:28 -0800 Subject: [PATCH 53/83] Adding query logging to presto queries --- .../java/com/facebook/presto/execution/SqlQueryManager.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java index e004f2024012..52a0751cc3d1 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java @@ -306,6 +306,10 @@ public QueryInfo createQuery(Session session, String query) stats.queryFinished(info); queryMonitor.completionEvent(info); expirationQueue.add(queryExecution); + log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s", + info.getQueryId(), newValue, info.getErrorType(), info.getErrorCode(), + session.getUser(), info.getQueryStats().getElapsedTime(), + info.getQuery().replace(System.getProperty("line.separator"), " "))); } }); From 04e60cce32c2a06bf77818fb6b522c0ab0864eea Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 29 Feb 2016 14:38:01 -0800 Subject: [PATCH 54/83] Add a few logs in DiscoveryNodeManager to debug 'No worker nodes' error. --- .../facebook/presto/metadata/DiscoveryNodeManager.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java index f6d470e88352..7238f4f3f94b 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java @@ -26,6 +26,7 @@ import io.airlift.discovery.client.ServiceSelector; import io.airlift.discovery.client.ServiceType; import io.airlift.http.client.HttpClient; +import io.airlift.log.Logger; import io.airlift.node.NodeInfo; import io.airlift.units.Duration; @@ -60,6 +61,8 @@ public final class DiscoveryNodeManager implements InternalNodeManager { + private static final Logger log = Logger.get(DiscoveryNodeManager.class); + private static final Duration MAX_AGE = new Duration(5, TimeUnit.SECONDS); private static final Splitter DATASOURCES_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); @@ -124,12 +127,18 @@ public void startPollingNodeStates() Set deadNodes = difference(nodeStates.keySet(), aliveNodeIds).immutableCopy(); nodeStates.keySet().removeAll(deadNodes); + if (deadNodes.size() > 0) { + log.warn("Dead nodes: %s", deadNodes); + } + // Add new nodes for (Node node : aliveNodes) { nodeStates.putIfAbsent(node.getNodeIdentifier(), new RemoteNodeState(httpClient, uriBuilderFrom(node.getHttpUri()).appendPath("/v1/info/state").build())); } + log.debug("Number of alive nodes: %d", nodeStates.size()); + // Schedule refresh nodeStates.values().forEach(RemoteNodeState::asyncRefresh); }, 1, 5, TimeUnit.SECONDS); From 4ca01a760e2610e19562eea4c4e2e5ff749283c9 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 29 Feb 2016 15:27:05 -0800 Subject: [PATCH 55/83] Upgrade to 0.139-tw-0.18 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 632225d50fe1..9f3597db7a77 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.139-tw-0.17 + 0.139-tw-0.18 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index d84fe856c920..1b0a10a8cbd6 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 80781a476f81..1629008c4a62 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 073888296318..671b065d02f8 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.17 + 0.139-tw-0.18 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8117fa6549ae..fa8e9ef3fc0e 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 5493f10852bd..031b25c708c4 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 4339db66dc64..b9c5632037af 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index dd2800d32a73..a8b37a95d835 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 907443ee18ec..23b87a6a0851 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 7df4d2893179..0033399e5eb7 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 85b2c4f1524d..9a212eb99757 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 1c254fb609c8..fafbe305fc71 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 0e58e0b8a334..f2ef547e83e6 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index f75f642ef484..9b784c66bf4b 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 22e73c96b1d0..a984ac81da6e 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index cc32b3878947..02f321b1d5e3 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9f748e0114ee..9f1258948e5d 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 8be6aa536b7c..09d2e4ec341c 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index ef29485ce5c9..cc0875536dae 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 749ca56c4191..498eaaabdfc8 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 34b47c0df123..a53264ad7f8b 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 7754bb50b25c..9b547152999b 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 422be0c545d9..7ded555176ad 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 48218aa78fb5..518f2dd91b55 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 5891c6b4c875..aed0b1c31591 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 6c78d18b97ca..6b1a9d92df35 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.17 + 0.139-tw-0.18 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 8504a4df3bc8..e96cab0b52f0 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 0719269862f1..88ea917ca2d8 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index e15a8e9be8c5..e38908baa260 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 23bd57d1aef9..94886cb04ccc 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index eb130f83e50d..84c31bc33f0f 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index f23575bcf202..edfa5063e36a 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 85bc734c1bf3..4c737d4e9be3 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 97a45f3fce0f..83afc4fe5eaa 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 75edb19713ae..aac5464df7de 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.17 + 0.139-tw-0.18 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 730294962ce5..539fb12adb9a 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 507e3ebe0c51..d81371230049 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-verifier From f1085e08d059e10ec18706e1a0927e31bce34edc Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 1 Mar 2016 12:16:25 -0800 Subject: [PATCH 56/83] Look up parquet columns by name case-insensitive --- .../facebook/presto/hive/parquet/ParquetTypeUtils.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java index bcf9027daeba..0fce1d81eced 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.HiveColumnHandle; import parquet.schema.MessageType; +import parquet.schema.Type; public final class ParquetTypeUtils { @@ -28,6 +29,13 @@ public static parquet.schema.Type getParquetType(HiveColumnHandle column, Messag if (messageType.containsField(column.getName())) { return messageType.getType(column.getName()); } + // parquet is case-sensitive, but hive is not. all hive columns get converted to lowercase + // check for direct match above but if no match found, try case-insensitive match + for (Type type : messageType.getFields()) { + if (type.getName().equalsIgnoreCase(column.getName())) { + return type; + } + } return null; } From 5b4647a1bb4e8638d38e9e376469bec5ae26ee27 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 1 Mar 2016 16:47:02 -0800 Subject: [PATCH 57/83] Upgrading to 0.139-tw-0.19 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 9f3597db7a77..f9431fab7e4c 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.139-tw-0.18 + 0.139-tw-0.19 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 1b0a10a8cbd6..48bdb7c717c6 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 1629008c4a62..9b83d6abcd3d 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 671b065d02f8..55598291a33c 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.18 + 0.139-tw-0.19 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index fa8e9ef3fc0e..9491d2c60eaa 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 031b25c708c4..47d7c6fef678 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index b9c5632037af..c4fd099e44d0 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index a8b37a95d835..9ae21a132d6f 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 23b87a6a0851..64003da568a8 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0033399e5eb7..508ba4cca260 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 9a212eb99757..5bb50b9fedf9 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index fafbe305fc71..85a7d855d03d 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index f2ef547e83e6..c796d9bc36d5 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 9b784c66bf4b..8784ec38bc75 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index a984ac81da6e..83cf942d5b8d 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 02f321b1d5e3..19c29f7401b8 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9f1258948e5d..4d3016e99c94 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 09d2e4ec341c..a85718e4708a 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index cc0875536dae..1fe37919474c 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 498eaaabdfc8..b04fed59a0ff 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index a53264ad7f8b..5359608986e9 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 9b547152999b..30a708b8d147 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 7ded555176ad..74512ff0e849 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 518f2dd91b55..0d57737392af 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index aed0b1c31591..6384f163283e 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 6b1a9d92df35..693d2bb9c135 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.18 + 0.139-tw-0.19 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index e96cab0b52f0..2952839013c2 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 88ea917ca2d8..53c00bb94f9a 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index e38908baa260..490cd8218967 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 94886cb04ccc..da1094e39d4b 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 84c31bc33f0f..de827fc9fd70 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index edfa5063e36a..33c89198a37c 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 4c737d4e9be3..c627552ef199 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 83afc4fe5eaa..083bb3564d3c 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index aac5464df7de..0d21f45e190e 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.18 + 0.139-tw-0.19 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 539fb12adb9a..b10f3a011cab 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index d81371230049..1a30c0800d0d 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-verifier From f766a0c616d437eca61bb9b4f1d9e7712373341c Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 1 Mar 2016 18:07:59 -0800 Subject: [PATCH 58/83] Handle hive keywords when doing a name-based parquet field lookup --- .../presto/hive/parquet/ParquetTypeUtils.java | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java index 0fce1d81eced..8891fb88afcd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java @@ -26,17 +26,15 @@ private ParquetTypeUtils() public static parquet.schema.Type getParquetType(HiveColumnHandle column, MessageType messageType, boolean useParquetColumnNames) { if (useParquetColumnNames) { - if (messageType.containsField(column.getName())) { - return messageType.getType(column.getName()); - } - // parquet is case-sensitive, but hive is not. all hive columns get converted to lowercase - // check for direct match above but if no match found, try case-insensitive match - for (Type type : messageType.getFields()) { - if (type.getName().equalsIgnoreCase(column.getName())) { - return type; - } + String name = column.getName(); + Type type = getParquetTypeByName(name, messageType); + + // when a parquet field is a hive keyword we append an _ to it in hive. When doing + // a name-based lookup, we need to strip it off again if we didn't get a direct match. + if (type == null && name.endsWith("_")) { + type = getParquetTypeByName(name.substring(0, name.length() - 1), messageType); } - return null; + return type; } if (column.getHiveColumnIndex() < messageType.getFieldCount()) { @@ -44,4 +42,20 @@ public static parquet.schema.Type getParquetType(HiveColumnHandle column, Messag } return null; } + + private static parquet.schema.Type getParquetTypeByName(String columnName, MessageType messageType) + { + if (messageType.containsField(columnName)) { + return messageType.getType(columnName); + } + // parquet is case-sensitive, but hive is not. all hive columns get converted to lowercase + // check for direct match above but if no match found, try case-insensitive match + for (Type type : messageType.getFields()) { + if (type.getName().equalsIgnoreCase(columnName)) { + return type; + } + } + + return null; + } } From ca9961e8deeca4f1640673511870fbb08d908b72 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 1 Mar 2016 20:40:26 -0800 Subject: [PATCH 59/83] Refactor to make diff cleaner --- .../presto/hive/parquet/ParquetTypeUtils.java | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java index 8891fb88afcd..bdd315f17218 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java @@ -26,15 +26,7 @@ private ParquetTypeUtils() public static parquet.schema.Type getParquetType(HiveColumnHandle column, MessageType messageType, boolean useParquetColumnNames) { if (useParquetColumnNames) { - String name = column.getName(); - Type type = getParquetTypeByName(name, messageType); - - // when a parquet field is a hive keyword we append an _ to it in hive. When doing - // a name-based lookup, we need to strip it off again if we didn't get a direct match. - if (type == null && name.endsWith("_")) { - type = getParquetTypeByName(name.substring(0, name.length() - 1), messageType); - } - return type; + return findParquetTypeByName(column, messageType); } if (column.getHiveColumnIndex() < messageType.getFieldCount()) { @@ -43,6 +35,28 @@ public static parquet.schema.Type getParquetType(HiveColumnHandle column, Messag return null; } + /** + * Find the column type by name using returning the first match with the following logic: + *
    + *
  • direct match
    • + *
  • case-insensitive match
    • + *
  • if the name ends with _, remove it and direct match
    • + *
  • if the name ends with _, remove it and case-insensitive match
    • + *
+ */ + private static parquet.schema.Type findParquetTypeByName(HiveColumnHandle column, MessageType messageType) + { + String name = column.getName(); + Type type = getParquetTypeByName(name, messageType); + + // when a parquet field is a hive keyword we append an _ to it in hive. When doing + // a name-based lookup, we need to strip it off again if we didn't get a direct match. + if (type == null && name.endsWith("_")) { + type = getParquetTypeByName(name.substring(0, name.length() - 1), messageType); + } + return type; + } + private static parquet.schema.Type getParquetTypeByName(String columnName, MessageType messageType) { if (messageType.containsField(columnName)) { From d7d66a28954049e9d41baf0d7eefd0bfd245637e Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 11 Mar 2016 11:25:21 -0800 Subject: [PATCH 60/83] Upgrade to 0.141. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index f9431fab7e4c..1cc3360c33ef 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.139-tw-0.19 + 0.141 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 48bdb7c717c6..aefbe525d11c 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 9b83d6abcd3d..883a041146d2 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 55598291a33c..546f6323162c 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.19 + 0.141 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 9491d2c60eaa..7637c11bcb13 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 47d7c6fef678..f24bb402f6f5 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index c4fd099e44d0..db5650329053 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 9ae21a132d6f..755f4b0c0bd0 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 64003da568a8..1bff918f4e79 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 508ba4cca260..ad3c2de908f1 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 5bb50b9fedf9..5bd58c4889a2 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 85a7d855d03d..360175ba7f1f 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index c796d9bc36d5..fc5b3c6bb0cc 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 8784ec38bc75..2e2d0c2e6938 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 83cf942d5b8d..f9cedd62b887 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 19c29f7401b8..77c66becb7c1 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 4d3016e99c94..80c89914cb60 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index a85718e4708a..8271359203ce 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 1fe37919474c..057819d9734b 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index b04fed59a0ff..ecbbd667febc 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 5359608986e9..1aa7767621ae 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 30a708b8d147..f02eced794de 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 74512ff0e849..906c40b97ba9 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 0d57737392af..b30f2e739442 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 6384f163283e..8f130d06d809 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 693d2bb9c135..ec7b54712bed 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.19 + 0.141 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 2952839013c2..17601b1b1823 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 53c00bb94f9a..f6f597487812 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 490cd8218967..31eb7a942001 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index da1094e39d4b..eb44ebac42f7 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index de827fc9fd70..67b64fdc2156 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 33c89198a37c..5b61cec72b21 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index c627552ef199..e9ab9fba2bba 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 083bb3564d3c..05bdf43a0a08 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 0d21f45e190e..696256a3122b 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.19 + 0.141 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index b10f3a011cab..72aaa2acb3dd 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 1a30c0800d0d..5f6c00cf18f5 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-verifier From 94bcf3e46c886684e45a5a796d0ae9167e5ad6f8 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 11 Mar 2016 11:29:59 -0800 Subject: [PATCH 61/83] Upgrade to 0.141-tw-0.20. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 411996e53ee0..130f0ff9eb9d 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.141 + 0.141-tw-0.20 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index def710c5c2c6..00cc629d364c 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 883a041146d2..3652e17bd06f 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 546f6323162c..8f86dabd3e25 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141 + 0.141-tw-0.20 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 7637c11bcb13..1c0d41086973 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index f24bb402f6f5..1bdeaecdcea2 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index db5650329053..4f0675eda340 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 755f4b0c0bd0..bdcff6cdb591 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 1bff918f4e79..f54c94db8398 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index ad3c2de908f1..4b4ad310d3e1 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 5bd58c4889a2..2c4cc361f6fb 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 360175ba7f1f..7bb7379f70b6 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index fc5b3c6bb0cc..09e13a19e68d 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 2e2d0c2e6938..6e304d8b10bd 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index f9cedd62b887..cfa615c8d2dc 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 77c66becb7c1..cebd8614c068 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 80c89914cb60..24ca263be84e 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 8271359203ce..4ae769912294 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 057819d9734b..31620ae9fe5a 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index ecbbd667febc..b1eed0066f04 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 1aa7767621ae..b5e67298855a 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index f02eced794de..4722581cd3a5 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 906c40b97ba9..d710b58886f5 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index b30f2e739442..830af643de31 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 8f130d06d809..c8e057ed14aa 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index ec7b54712bed..bfe2383a567f 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141 + 0.141-tw-0.20 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 17601b1b1823..3b527cbaa5dc 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index f6f597487812..5c724e169c0c 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 31eb7a942001..4c226d7edc39 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index eb44ebac42f7..14038c648f0f 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 67b64fdc2156..2c4b2c1f49eb 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 5b61cec72b21..a977f2d08420 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index e9ab9fba2bba..301e4c4efe55 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 05bdf43a0a08..dc8b11426992 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 696256a3122b..25c2414ce83f 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141 + 0.141-tw-0.20 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 72aaa2acb3dd..3979e5a18555 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 5f6c00cf18f5..9dbeee8b8252 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-verifier From 3de1ca483acb8944e714631df79fe62a3023f450 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 09:35:41 -0700 Subject: [PATCH 62/83] Use modules and query events for logging --- .../presto/execution/SqlQueryManager.java | 4 - .../facebook/presto/server/PrestoServer.java | 2 +- .../facebook/presto/twitter/ModuleLoader.java | 37 ++++++++ .../presto/twitter/logging/QueryLogger.java | 91 +++++++++++++++++++ 4 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java index 51e261aeeb10..6722c701fd5e 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java @@ -309,10 +309,6 @@ public QueryInfo createQuery(Session session, String query) stats.queryFinished(info); queryMonitor.completionEvent(info); expirationQueue.add(queryExecution); - log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s", - info.getQueryId(), newValue, info.getErrorType(), info.getErrorCode(), - session.getUser(), info.getQueryStats().getElapsedTime(), - info.getQuery().replace(System.getProperty("line.separator"), " "))); } }); diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index de138dc25cba..685a0454c73a 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -154,7 +154,7 @@ public void run() protected Iterable getAdditionalModules() { - return ImmutableList.of(); + return com.facebook.presto.twitter.ModuleLoader.getAdditionalModules(); } private static void updateDatasources(Announcer announcer, Metadata metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java new file mode 100644 index 000000000000..aee0a6ca2825 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter; + +import com.facebook.presto.twitter.logging.QueryLogger; +import com.google.common.collect.ImmutableList; +import com.google.inject.Module; +import com.google.inject.Scopes; +import com.google.inject.multibindings.Multibinder; +import io.airlift.event.client.EventClient; + +public class ModuleLoader +{ + private ModuleLoader() + { + } + + public static Iterable getAdditionalModules() + { + return ImmutableList.of(binder -> + Multibinder.newSetBinder(binder, EventClient.class) + .addBinding() + .to(QueryLogger.class) + .in(Scopes.SINGLETON)); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java new file mode 100644 index 000000000000..863847ab73c0 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -0,0 +1,91 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.logging; + +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.spi.StandardErrorCode; +import io.airlift.event.client.AbstractEventClient; +import io.airlift.event.client.EventType; +import io.airlift.log.Logger; +import io.airlift.units.Duration; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +public class QueryLogger extends AbstractEventClient +{ + private static final int MAX_QUERY_LENGTH = 1000; + private static final String DASH = "-"; + private static final String COLON = ":"; + private static final String SPACE = " "; + private static final String ELIPSIS = "..."; + + private static final Logger log = Logger.get(QueryLogger.class); + + @Override + protected void postEvent(T event) + throws IOException + { + EventType eventTypeAnnotation = event.getClass().getAnnotation(EventType.class); + if (eventTypeAnnotation == null) { + return; + } + + // other event types exist, like QueryCreatedEvent and SplitCompletionEvent + if (eventTypeAnnotation.value().equals("QueryCompletion")) { + logQueryComplete((QueryCompletionEvent) event); + } + } + + private static void logQueryComplete(QueryCompletionEvent event) + { + String errorType = DASH; + String errorCode = DASH; + if (event.getErrorCode() != null) { + errorType = StandardErrorCode.toErrorType(event.getErrorCode()).toString(); + if (event.getErrorCodeName() != null) { + errorCode = event.getErrorCodeName() + COLON + event.getErrorCode(); + } + } + + Duration duration = (new Duration( + event.getEndTime().getMillis() - + event.getCreateTime().getMillis(), TimeUnit.MILLISECONDS)) + .convertToMostSuccinctTimeUnit(); + + log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", + event.getQueryId(), toLogValue(event.getRemoteClientAddress()), + event.getQueryState(), errorType, errorCode, + event.getUser(), duration, + cleanseAndTrimQuery(event.getQuery()))); + } + + private static String toLogValue(Object object) + { + if (object == null) { + return DASH; + } + else { + return object.toString(); + } + } + + private static String cleanseAndTrimQuery(String query) + { + if (query.length() > MAX_QUERY_LENGTH) { + query = query.substring(0, MAX_QUERY_LENGTH) + ELIPSIS; + } + return query.replace(System.getProperty("line.separator"), SPACE); + } +} From f055db69eaf6901002e0d017e85223cc5cdede80 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 10:25:55 -0700 Subject: [PATCH 63/83] Add javadocs --- .../facebook/presto/server/PrestoServer.java | 3 +- .../facebook/presto/twitter/ModuleLoader.java | 37 ------------------- .../presto/twitter/logging/QueryLogger.java | 3 ++ 3 files changed, 5 insertions(+), 38 deletions(-) delete mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 685a0454c73a..78b01e8cd4fb 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -24,6 +24,7 @@ import com.facebook.presto.security.AccessControlModule; import com.facebook.presto.server.security.ServerSecurityModule; import com.facebook.presto.sql.parser.SqlParserOptions; +import com.facebook.presto.twitter.TwitterModuleLoader; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; @@ -154,7 +155,7 @@ public void run() protected Iterable getAdditionalModules() { - return com.facebook.presto.twitter.ModuleLoader.getAdditionalModules(); + return TwitterModuleLoader.getAdditionalModules(); } private static void updateDatasources(Announcer announcer, Metadata metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java deleted file mode 100644 index aee0a6ca2825..000000000000 --- a/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.twitter; - -import com.facebook.presto.twitter.logging.QueryLogger; -import com.google.common.collect.ImmutableList; -import com.google.inject.Module; -import com.google.inject.Scopes; -import com.google.inject.multibindings.Multibinder; -import io.airlift.event.client.EventClient; - -public class ModuleLoader -{ - private ModuleLoader() - { - } - - public static Iterable getAdditionalModules() - { - return ImmutableList.of(binder -> - Multibinder.newSetBinder(binder, EventClient.class) - .addBinding() - .to(QueryLogger.class) - .in(Scopes.SINGLETON)); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java index 863847ab73c0..b70eda37da7c 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -23,6 +23,9 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; +/** + * Class that listens for query completion events and logs them to a file + */ public class QueryLogger extends AbstractEventClient { private static final int MAX_QUERY_LENGTH = 1000; From 3c1c7b661cf0b79abeb33448eafd9b0fe2c421b0 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 10:45:11 -0700 Subject: [PATCH 64/83] Add javadocs --- .../presto/twitter/TwitterModuleLoader.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java new file mode 100644 index 000000000000..44bfc995b462 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter; + +import com.facebook.presto.twitter.logging.QueryLogger; +import com.google.common.collect.ImmutableList; +import com.google.inject.Module; +import com.google.inject.Scopes; +import com.google.inject.multibindings.Multibinder; +import io.airlift.event.client.EventClient; + +/** + * Loader that initializes custom Twitter code to inject into Presto. Whenever + * possible we should use this pattern to inject custom functionality, since it + * makes it easier to differentiate our patches from the core OS code. + * + * If the functionality we wish to add/override isn't currently possible to via + * overriding a guice module, we should contribute the necessary modules/interfaces + * into the OS Presto code base to make it possible. + */ +public class TwitterModuleLoader +{ + private TwitterModuleLoader() + { + } + + public static Iterable getAdditionalModules() + { + return ImmutableList.of(binder -> + Multibinder.newSetBinder(binder, EventClient.class) + .addBinding() + .to(QueryLogger.class) + .in(Scopes.SINGLETON)); + } +} From d8afd6223cc9d99d3cfd430b0752b53c0cb66c75 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 10:47:05 -0700 Subject: [PATCH 65/83] fix imports --- .../src/main/java/com/facebook/presto/server/PrestoServer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 78b01e8cd4fb..4bac276668ef 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -24,7 +24,6 @@ import com.facebook.presto.security.AccessControlModule; import com.facebook.presto.server.security.ServerSecurityModule; import com.facebook.presto.sql.parser.SqlParserOptions; -import com.facebook.presto.twitter.TwitterModuleLoader; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; @@ -155,7 +154,7 @@ public void run() protected Iterable getAdditionalModules() { - return TwitterModuleLoader.getAdditionalModules(); + return com.facebook.presto.twitter.TwitterModuleLoader.getAdditionalModules(); } private static void updateDatasources(Announcer announcer, Metadata metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) From 251db4d7c15df61b8e86eb1bd3e21ad904ecac04 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 11:01:00 -0700 Subject: [PATCH 66/83] add splits, rows and bytes --- .../com/facebook/presto/twitter/logging/QueryLogger.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java index b70eda37da7c..1ad52efb0d67 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -67,10 +67,10 @@ private static void logQueryComplete(QueryCompletionEvent event) event.getCreateTime().getMillis(), TimeUnit.MILLISECONDS)) .convertToMostSuccinctTimeUnit(); - log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", + log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", event.getQueryId(), toLogValue(event.getRemoteClientAddress()), - event.getQueryState(), errorType, errorCode, - event.getUser(), duration, + event.getQueryState(), errorType, errorCode, event.getUser(), duration, + event.getSplits(), event.getTotalRows(), event.getTotalBytes(), cleanseAndTrimQuery(event.getQuery()))); } From 95035ecf3eb7a1a9569fa9a13a15b00463992349 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 11:56:17 -0700 Subject: [PATCH 67/83] Change to use QueryComplete --- .../com/facebook/presto/twitter/logging/QueryLogger.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java index 1ad52efb0d67..0a6f5c3e105c 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -33,6 +33,7 @@ public class QueryLogger extends AbstractEventClient private static final String COLON = ":"; private static final String SPACE = " "; private static final String ELIPSIS = "..."; + private static final String QUERY_COMPLETION = "QueryCompletion"; private static final Logger log = Logger.get(QueryLogger.class); @@ -46,7 +47,7 @@ protected void postEvent(T event) } // other event types exist, like QueryCreatedEvent and SplitCompletionEvent - if (eventTypeAnnotation.value().equals("QueryCompletion")) { + if (eventTypeAnnotation.value().equals(QUERY_COMPLETION)) { logQueryComplete((QueryCompletionEvent) event); } } @@ -67,8 +68,8 @@ private static void logQueryComplete(QueryCompletionEvent event) event.getCreateTime().getMillis(), TimeUnit.MILLISECONDS)) .convertToMostSuccinctTimeUnit(); - log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", - event.getQueryId(), toLogValue(event.getRemoteClientAddress()), + log.info(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", + QUERY_COMPLETION, event.getQueryId(), toLogValue(event.getRemoteClientAddress()), event.getQueryState(), errorType, errorCode, event.getUser(), duration, event.getSplits(), event.getTotalRows(), event.getTotalBytes(), cleanseAndTrimQuery(event.getQuery()))); From dbb4c42e330e42815a097da727f26cad68d55141 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 12:29:07 -0700 Subject: [PATCH 68/83] Add event process and handlers and scribe query completion events --- pom.xml | 55 ++++++ presto-main/pom.xml | 18 ++ .../facebook/presto/event/EventProcessor.java | 90 ++++++++++ .../event/query/QueryCompletionEvent.java | 2 +- .../presto/event/query/QueryCreatedEvent.java | 2 +- .../presto/event/query/QueryEvent.java | 22 +++ .../presto/event/query/QueryEventHandler.java | 19 ++ .../event/query/SplitCompletionEvent.java | 2 +- .../presto/twitter/TwitterModuleLoader.java | 26 ++- .../presto/twitter/logging/QueryLogger.java | 27 +-- .../presto/twitter/logging/QueryScriber.java | 169 ++++++++++++++++++ 11 files changed, 403 insertions(+), 29 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java create mode 100644 presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java create mode 100644 presto-main/src/main/java/com/facebook/presto/event/query/QueryEventHandler.java create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java diff --git a/pom.xml b/pom.xml index 130f0ff9eb9d..ae36b8cf3d9c 100644 --- a/pom.xml +++ b/pom.xml @@ -730,6 +730,61 @@ hive-apache-jdbc 0.13.1-1 + + + + com.twitter + presto-thrift-java + 0.0.1 + + + com.twitter + util-core_2.11 + + + com.twitter + util-core-java + + + com.twitter + util-function_2.10 + + + com.twitter + util-function-java + + + commons-logging + commons-logging + + + org.scala-lang.modules + scala-parser-combinators_2.11 + + + + + com.twitter + util-logging_2.11 + 6.33.0 + + + commons-logging + commons-logging + + + + + org.scala-lang + scala-library + 2.11.7 + + + commons-logging + commons-logging + + + diff --git a/presto-main/pom.xml b/presto-main/pom.xml index b1eed0066f04..dfe5c5d08d7a 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -296,6 +296,24 @@ tpch test + + + + com.twitter + presto-thrift-java + + + com.twitter + util-logging_2.11 + + + org.apache.thrift + libthrift + + + org.scala-lang + scala-library + diff --git a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java new file mode 100644 index 000000000000..edaabc5437de --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.event; + +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryCreatedEvent; +import com.facebook.presto.event.query.QueryEvent; +import com.facebook.presto.event.query.QueryEventHandler; +import com.facebook.presto.event.query.SplitCompletionEvent; +import com.facebook.presto.twitter.logging.QueryLogger; +import com.google.inject.Inject; +import io.airlift.event.client.AbstractEventClient; +import io.airlift.event.client.EventType; +import io.airlift.log.Logger; + +import java.io.IOException; +import java.util.Set; + +public class EventProcessor extends AbstractEventClient +{ + private static final String QUERY_CREATED = "QueryCreated"; + private static final String QUERY_COMPLETION = "QueryCompletion"; + private static final String SPLIT_COMPLETION = "SplitCompletion"; + private static final Logger log = Logger.get(QueryLogger.class); + + private Set> queryCreatedEventHandlers; + private Set> queryCompletionEventHandlers; + private Set> splitCompletionEventHandlers; + + @Inject + public EventProcessor( + Set> queryCreatedEventHandlers, + Set> queryCompletionEventHandlers, + Set> splitCompletionEventHandlers) + { + this.queryCreatedEventHandlers = queryCreatedEventHandlers; + this.queryCompletionEventHandlers = queryCompletionEventHandlers; + this.splitCompletionEventHandlers = splitCompletionEventHandlers; + } + + @Override + protected void postEvent(T event) + throws IOException + { + EventType eventTypeAnnotation = event.getClass().getAnnotation(EventType.class); + if (eventTypeAnnotation == null) { + return; + } + + String type = eventTypeAnnotation.value(); + + switch (type) { + case QUERY_CREATED: + handle(queryCreatedEventHandlers, type, (QueryCreatedEvent) event); + break; + case QUERY_COMPLETION: + handle(queryCompletionEventHandlers, type, (QueryCompletionEvent) event); + break; + case SPLIT_COMPLETION: + handle(splitCompletionEventHandlers, type, (SplitCompletionEvent) event); + break; + default: + log.warn("Unrecognized event found: " + type); + } + } + + private void handle(Set> handlers, String type, E event) + { + for (QueryEventHandler handler : handlers) { + try { + handler.handle(event); + } + catch (Throwable e) { + log.error(e, String.format( + "Exception processing %s event for query %s", type, event.getQueryId())); + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryCompletionEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryCompletionEvent.java index c005649e77e2..3c95ffe66ef0 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/query/QueryCompletionEvent.java +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryCompletionEvent.java @@ -31,7 +31,7 @@ @Immutable @EventType("QueryCompletion") -public class QueryCompletionEvent +public class QueryCompletionEvent implements QueryEvent { private final QueryId queryId; private final String transactionId; diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryCreatedEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryCreatedEvent.java index 64871433ecee..6be296bba8e7 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/query/QueryCreatedEvent.java +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryCreatedEvent.java @@ -24,7 +24,7 @@ @Immutable @EventType("QueryCreated") -public class QueryCreatedEvent +public class QueryCreatedEvent implements QueryEvent { private final QueryId queryId; private final String transactionId; diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java new file mode 100644 index 000000000000..33ca974b826c --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.event.query; + +/** + * + */ +public interface QueryEvent +{ + String getQueryId(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryEventHandler.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEventHandler.java new file mode 100644 index 000000000000..2aa0bd381a5f --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEventHandler.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.event.query; + +public interface QueryEventHandler +{ + void handle(T event); +} diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/SplitCompletionEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/SplitCompletionEvent.java index f9374c21b09d..4a9e206b9962 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/query/SplitCompletionEvent.java +++ b/presto-main/src/main/java/com/facebook/presto/event/query/SplitCompletionEvent.java @@ -29,7 +29,7 @@ @Immutable @EventType("SplitCompletion") -public class SplitCompletionEvent +public class SplitCompletionEvent implements QueryEvent { private final QueryId queryId; private final StageId stageId; diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java index 44bfc995b462..eb23a7650a09 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java @@ -13,10 +13,17 @@ */ package com.facebook.presto.twitter; +import com.facebook.presto.event.EventProcessor; +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryCreatedEvent; +import com.facebook.presto.event.query.QueryEventHandler; +import com.facebook.presto.event.query.SplitCompletionEvent; import com.facebook.presto.twitter.logging.QueryLogger; +import com.facebook.presto.twitter.logging.QueryScriber; import com.google.common.collect.ImmutableList; import com.google.inject.Module; import com.google.inject.Scopes; +import com.google.inject.TypeLiteral; import com.google.inject.multibindings.Multibinder; import io.airlift.event.client.EventClient; @@ -37,10 +44,21 @@ private TwitterModuleLoader() public static Iterable getAdditionalModules() { - return ImmutableList.of(binder -> - Multibinder.newSetBinder(binder, EventClient.class) + return ImmutableList.of( + binder -> Multibinder.newSetBinder(binder, EventClient.class) .addBinding() - .to(QueryLogger.class) - .in(Scopes.SINGLETON)); + .to(EventProcessor.class) + .in(Scopes.SINGLETON), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) + .addBinding() + .to(new TypeLiteral(){}) + .in(Scopes.SINGLETON), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) + .addBinding() + .to(QueryScriber.class) + .in(Scopes.SINGLETON) + ); } } diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java index 0a6f5c3e105c..02edd8da2440 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -14,19 +14,17 @@ package com.facebook.presto.twitter.logging; import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryEventHandler; import com.facebook.presto.spi.StandardErrorCode; -import io.airlift.event.client.AbstractEventClient; -import io.airlift.event.client.EventType; import io.airlift.log.Logger; import io.airlift.units.Duration; -import java.io.IOException; import java.util.concurrent.TimeUnit; /** - * Class that listens for query completion events and logs them to a file + * Class that logs query events to a file */ -public class QueryLogger extends AbstractEventClient +public class QueryLogger implements QueryEventHandler { private static final int MAX_QUERY_LENGTH = 1000; private static final String DASH = "-"; @@ -38,21 +36,7 @@ public class QueryLogger extends AbstractEventClient private static final Logger log = Logger.get(QueryLogger.class); @Override - protected void postEvent(T event) - throws IOException - { - EventType eventTypeAnnotation = event.getClass().getAnnotation(EventType.class); - if (eventTypeAnnotation == null) { - return; - } - - // other event types exist, like QueryCreatedEvent and SplitCompletionEvent - if (eventTypeAnnotation.value().equals(QUERY_COMPLETION)) { - logQueryComplete((QueryCompletionEvent) event); - } - } - - private static void logQueryComplete(QueryCompletionEvent event) + public void handle(QueryCompletionEvent event) { String errorType = DASH; String errorCode = DASH; @@ -64,8 +48,7 @@ private static void logQueryComplete(QueryCompletionEvent event) } Duration duration = (new Duration( - event.getEndTime().getMillis() - - event.getCreateTime().getMillis(), TimeUnit.MILLISECONDS)) + event.getQueryWallTimeMs(), TimeUnit.MILLISECONDS)) .convertToMostSuccinctTimeUnit(); log.info(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java new file mode 100644 index 000000000000..0110afdf8a61 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -0,0 +1,169 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.logging; + +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryEventHandler; +import com.google.common.base.Optional; +import com.twitter.logging.BareFormatter$; +import com.twitter.logging.Level; +import com.twitter.logging.QueueingHandler; +import com.twitter.logging.ScribeHandler; +import com.twitter.presto.thriftjava.QueryState; +import io.airlift.log.Logger; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; + +import java.util.Base64; +import java.util.logging.LogRecord; + +/** + * Class that scribes query completion events + */ +public class QueryScriber implements QueryEventHandler +{ + private static final String SCRIBE_CATEGORY = "test_presto_query_complete"; + private static final int MAX_QUEUE_SIZE = 1000; + + private static final Logger log = Logger.get(QueryScriber.class); + + private QueueingHandler queueingHandler; + + // TSerializer is not thread safe + private final ThreadLocal serializer = new ThreadLocal() + { + @Override protected TSerializer initialValue() + { + return new TSerializer(); + } + }; + + public QueryScriber() + { + ScribeHandler scribeHandler = new ScribeHandler( + ScribeHandler.DefaultHostname(), + ScribeHandler.DefaultPort(), + SCRIBE_CATEGORY, + ScribeHandler.DefaultBufferTime(), + ScribeHandler.DefaultConnectBackoff(), + ScribeHandler.DefaultMaxMessagesPerTransaction(), + ScribeHandler.DefaultMaxMessagesToBuffer(), + BareFormatter$.MODULE$, + scala.Option.apply((Level) null)); + queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); + } + + @Override + public void handle(QueryCompletionEvent event) + { + com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); + Optional message = serializeThriftToString(thriftEvent); + + if (message.isPresent()) { + LogRecord logRecord = new LogRecord(Level.ALL, message.get()); + queueingHandler.publish(logRecord); + } + else { + log.warn("Unable to serialize QueryCompletionEvent: " + event); + } + } + + /** + * Serialize a thrift object to bytes, compress, then encode as a base64 string. + */ + private Optional serializeThriftToString(TBase thriftMessage) + { + try { + return Optional.of( + Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage))); + } + catch (TException e) { + log.warn(e, "Could not serialize thrift object" + thriftMessage); + return Optional.absent(); + } + } + + private static com.twitter.presto.thriftjava.QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletionEvent event) + { + com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = + new com.twitter.presto.thriftjava.QueryCompletionEvent(); + + thriftEvent.query_id = event.getQueryId(); + thriftEvent.transaction_id = event.getTransactionId(); + thriftEvent.user = event.getUser(); + thriftEvent.principal = event.getPrincipal(); + thriftEvent.source = event.getSource(); + thriftEvent.server_version = event.getServerVersion(); + thriftEvent.environment = event.getEnvironment(); + thriftEvent.catalog = event.getCatalog(); + thriftEvent.schema = event.getSchema(); + thriftEvent.remote_client_address = event.getRemoteClientAddress(); + thriftEvent.user_agent = event.getUserAgent(); + thriftEvent.query_state = QueryState.valueOf(event.getQueryState()); + thriftEvent.uri = event.getUri(); + thriftEvent.field_names = event.getFieldNames(); + thriftEvent.query = event.getQuery(); + thriftEvent.create_time_ms = event.getCreateTime().getMillis(); + thriftEvent.execution_start_time_ms = event.getExecutionStartTime().getMillis(); + thriftEvent.end_time_ms = event.getEndTime().getMillis(); + thriftEvent.queued_time_ms = event.getQueuedTimeMs(); + if (event.getAnalysisTimeMs() != null) { + thriftEvent.analysis_time_ms = event.getAnalysisTimeMs(); + } + if (event.getDistributedPlanningTimeMs() != null) { + thriftEvent.distributed_planning_time_ms = event.getDistributedPlanningTimeMs(); + } + if (event.getTotalSplitWallTimeMs() != null) { + thriftEvent.total_split_wall_time_ms = event.getTotalSplitWallTimeMs(); + } + if (event.getTotalSplitCpuTimeMs() != null) { + thriftEvent.total_split_cpu_time_ms = event.getTotalSplitCpuTimeMs(); + } + if (event.getTotalBytes() != null) { + thriftEvent.total_bytes = event.getTotalBytes(); + } + if (event.getTotalRows() != null) { + thriftEvent.total_rows = event.getTotalRows(); + } + thriftEvent.splits = event.getSplits(); + if (event.getErrorCode() != null) { + thriftEvent.error_code_id = event.getErrorCode(); + } + thriftEvent.error_code_name = event.getErrorCodeName(); + thriftEvent.failure_type = event.getFailureType(); + thriftEvent.failure_message = event.getFailureMessage(); + thriftEvent.failure_task = event.getFailureTask(); + thriftEvent.failure_host = event.getFailureHost(); + thriftEvent.output_stage_json = event.getOutputStageJson(); + thriftEvent.failures_json = event.getFailuresJson(); + thriftEvent.inputs_json = event.getInputsJson(); + thriftEvent.session_properties_json = event.getSessionPropertiesJson(); + thriftEvent.query_wall_time_ms = event.getQueryWallTimeMs(); + if (event.getBytesPerSec() != null) { + thriftEvent.bytes_per_sec = event.getBytesPerSec(); + } + if (event.getBytesPerCpuSec() != null) { + thriftEvent.bytes_per_cpu_sec = event.getBytesPerCpuSec(); + } + if (event.getRowsPerSec() != null) { + thriftEvent.rows_per_sec = event.getRowsPerSec(); + } + if (event.getRowsPerCpuSec() != null) { + thriftEvent.rows_per_cpu_sec = event.getRowsPerCpuSec(); + } + + return thriftEvent; + } +} From 38e6d09f7a30f27c21b2402bd5f0895ffbf08199 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 13:39:30 -0700 Subject: [PATCH 69/83] code clean-up --- .../facebook/presto/event/EventProcessor.java | 3 +++ .../facebook/presto/event/query/QueryEvent.java | 3 --- .../presto/twitter/logging/QueryScriber.java | 17 ++++++++--------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java index edaabc5437de..1ac55ea6d8a7 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java @@ -27,6 +27,9 @@ import java.io.IOException; import java.util.Set; +/** + * Class that listens for airlift events and sends presto events to handlers + */ public class EventProcessor extends AbstractEventClient { private static final String QUERY_CREATED = "QueryCreated"; diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java index 33ca974b826c..da43e387ee22 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java @@ -13,9 +13,6 @@ */ package com.facebook.presto.event.query; -/** - * - */ public interface QueryEvent { String getQueryId(); diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java index 0110afdf8a61..d4450aaf6a38 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.twitter.logging; -import com.facebook.presto.event.query.QueryCompletionEvent; import com.facebook.presto.event.query.QueryEventHandler; import com.google.common.base.Optional; import com.twitter.logging.BareFormatter$; import com.twitter.logging.Level; import com.twitter.logging.QueueingHandler; import com.twitter.logging.ScribeHandler; +import com.twitter.presto.thriftjava.QueryCompletionEvent; import com.twitter.presto.thriftjava.QueryState; import io.airlift.log.Logger; import org.apache.thrift.TBase; @@ -32,7 +32,7 @@ /** * Class that scribes query completion events */ -public class QueryScriber implements QueryEventHandler +public class QueryScriber implements QueryEventHandler { private static final String SCRIBE_CATEGORY = "test_presto_query_complete"; private static final int MAX_QUEUE_SIZE = 1000; @@ -66,17 +66,17 @@ public QueryScriber() } @Override - public void handle(QueryCompletionEvent event) + public void handle(com.facebook.presto.event.query.QueryCompletionEvent event) { com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); Optional message = serializeThriftToString(thriftEvent); if (message.isPresent()) { - LogRecord logRecord = new LogRecord(Level.ALL, message.get()); - queueingHandler.publish(logRecord); + LogRecord logRecord = new LogRecord(Level.ALL, message.get()); + queueingHandler.publish(logRecord); } else { - log.warn("Unable to serialize QueryCompletionEvent: " + event); + log.warn("Unable to serialize QueryCompletionEvent: " + event); } } @@ -95,10 +95,9 @@ private Optional serializeThriftToString(TBase thriftMessage) } } - private static com.twitter.presto.thriftjava.QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletionEvent event) + private static QueryCompletionEvent toThriftQueryCompletionEvent(com.facebook.presto.event.query.QueryCompletionEvent event) { - com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = - new com.twitter.presto.thriftjava.QueryCompletionEvent(); + QueryCompletionEvent thriftEvent = new QueryCompletionEvent(); thriftEvent.query_id = event.getQueryId(); thriftEvent.transaction_id = event.getTransactionId(); From 2ac4c4225977b175d2ecab181aa1f4d9a4cb1907 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 14:02:47 -0700 Subject: [PATCH 70/83] Fix package leakage --- .../main/java/com/facebook/presto/event/EventProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java index 1ac55ea6d8a7..c575a6a49818 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java @@ -18,7 +18,6 @@ import com.facebook.presto.event.query.QueryEvent; import com.facebook.presto.event.query.QueryEventHandler; import com.facebook.presto.event.query.SplitCompletionEvent; -import com.facebook.presto.twitter.logging.QueryLogger; import com.google.inject.Inject; import io.airlift.event.client.AbstractEventClient; import io.airlift.event.client.EventType; @@ -35,7 +34,7 @@ public class EventProcessor extends AbstractEventClient private static final String QUERY_CREATED = "QueryCreated"; private static final String QUERY_COMPLETION = "QueryCompletion"; private static final String SPLIT_COMPLETION = "SplitCompletion"; - private static final Logger log = Logger.get(QueryLogger.class); + private static final Logger log = Logger.get(EventProcessor.class); private Set> queryCreatedEventHandlers; private Set> queryCompletionEventHandlers; From a8d1ffc0bdadbd5bed81db42135f1ed2ea1479e5 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 15:15:11 -0700 Subject: [PATCH 71/83] Move twitter classes to a twitter package --- .../com/facebook/presto/hive/BackgroundHiveSplitLoader.java | 2 +- .../java/com/facebook/presto/hive/HiveConnectorFactory.java | 3 +++ .../com/facebook/presto/hive/HivePageSourceProvider.java | 2 +- .../{ => twitter}/hive/MetastoreStaticClusterModule.java | 5 ++++- .../{ => twitter}/hive/MetastoreZkDiscoveryBasedModule.java | 3 ++- .../presto/{ => twitter}/hive/ZookeeperMetastoreMonitor.java | 2 +- .../{ => twitter}/hive/ZookeeperServersetHiveCluster.java | 4 +++- .../hive/ZookeeperServersetMetastoreConfig.java | 2 +- .../facebook/presto/{ => twitter}/hive/util/UgiUtils.java | 2 +- .../{ => twitter}/hive/TestZookeeperMetastoreMonitor.java | 4 ++-- .../hive/TestZookeeperServersetMetastoreConfig.java | 2 +- .../facebook/presto/{ => twitter}/hive/util/TestUtils.java | 2 +- 12 files changed, 21 insertions(+), 12 deletions(-) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/MetastoreStaticClusterModule.java (84%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/MetastoreZkDiscoveryBasedModule.java (92%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/ZookeeperMetastoreMonitor.java (99%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/ZookeeperServersetHiveCluster.java (94%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/ZookeeperServersetMetastoreConfig.java (98%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/util/UgiUtils.java (97%) rename presto-hive/src/test/java/com/facebook/presto/{ => twitter}/hive/TestZookeeperMetastoreMonitor.java (98%) rename presto-hive/src/test/java/com/facebook/presto/{ => twitter}/hive/TestZookeeperServersetMetastoreConfig.java (98%) rename presto-hive/src/test/java/com/facebook/presto/{ => twitter}/hive/util/TestUtils.java (94%) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 676b856e6f7c..cd8d88e0d9e8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -16,10 +16,10 @@ import com.facebook.presto.hive.util.HiveFileIterator; import com.facebook.presto.hive.util.ResumableTask; import com.facebook.presto.hive.util.ResumableTasks; -import com.facebook.presto.hive.util.UgiUtils; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.twitter.hive.util.UgiUtils; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java index f544a553bdbb..7aa27c3ea11f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java @@ -29,6 +29,9 @@ import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.security.ConnectorAccessControl; import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.twitter.hive.MetastoreStaticClusterModule; +import com.facebook.presto.twitter.hive.MetastoreZkDiscoveryBasedModule; +import com.facebook.presto.twitter.hive.ZookeeperServersetMetastoreConfig; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableSet; import com.google.inject.Injector; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java index ff1c81f19367..1edee9f8d7d6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.hive; -import com.facebook.presto.hive.util.UgiUtils; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorPageSourceProvider; @@ -23,6 +22,7 @@ import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.twitter.hive.util.UgiUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreStaticClusterModule.java similarity index 84% rename from presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreStaticClusterModule.java index 18bfa3d1eff1..b8c1b7dc9a60 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreStaticClusterModule.java @@ -11,8 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; +import com.facebook.presto.hive.HiveCluster; +import com.facebook.presto.hive.StaticHiveCluster; +import com.facebook.presto.hive.StaticMetastoreConfig; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreZkDiscoveryBasedModule.java similarity index 92% rename from presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreZkDiscoveryBasedModule.java index a4d84813ab76..775a5afaf4c8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreZkDiscoveryBasedModule.java @@ -11,8 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; +import com.facebook.presto.hive.HiveCluster; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperMetastoreMonitor.java similarity index 99% rename from presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperMetastoreMonitor.java index 3789ca3c3271..e1d0f2011468 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperMetastoreMonitor.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; import com.google.common.net.HostAndPort; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java similarity index 94% rename from presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index 48f869ec8ed0..83642e41c50f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -11,8 +11,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; +import com.facebook.presto.hive.HiveCluster; +import com.facebook.presto.hive.HiveMetastoreClientFactory; import com.facebook.presto.hive.metastore.HiveMetastoreClient; import com.facebook.presto.spi.PrestoException; import com.google.common.net.HostAndPort; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java index 44c6f9d19188..26e36b469d0c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/UgiUtils.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/UgiUtils.java index 08f2dde30ff2..6d540bbe3c5a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/UgiUtils.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive.util; +package com.facebook.presto.twitter.hive.util; import org.apache.hadoop.security.UserGroupInformation; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java similarity index 98% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java rename to presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java index 6b82efa22f97..d0eba9f4e6e5 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java @@ -11,10 +11,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; import com.facebook.presto.hadoop.shaded.com.google.common.collect.ImmutableList; -import com.facebook.presto.hive.util.TestUtils; +import com.facebook.presto.twitter.hive.util.TestUtils; import com.google.common.net.HostAndPort; import io.airlift.log.Logger; import org.I0Itec.zkclient.ZkClient; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java similarity index 98% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java rename to presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java index bb1117697296..6992a752dea1 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/util/TestUtils.java similarity index 94% rename from presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java rename to presto-hive/src/test/java/com/facebook/presto/twitter/hive/util/TestUtils.java index 4315bac667c2..379ad3877e32 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/util/TestUtils.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive.util; +package com.facebook.presto.twitter.hive.util; import java.io.IOException; import java.net.ServerSocket; From 73c53029594b351bc9f593b657e6bd86c0475a82 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 15:26:47 -0700 Subject: [PATCH 72/83] Default package is presto and thrift is custom --- .../facebook/presto/twitter/logging/QueryScriber.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java index d4450aaf6a38..306029550ed6 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.twitter.logging; +import com.facebook.presto.event.query.QueryCompletionEvent; import com.facebook.presto.event.query.QueryEventHandler; import com.google.common.base.Optional; import com.twitter.logging.BareFormatter$; import com.twitter.logging.Level; import com.twitter.logging.QueueingHandler; import com.twitter.logging.ScribeHandler; -import com.twitter.presto.thriftjava.QueryCompletionEvent; import com.twitter.presto.thriftjava.QueryState; import io.airlift.log.Logger; import org.apache.thrift.TBase; @@ -32,7 +32,7 @@ /** * Class that scribes query completion events */ -public class QueryScriber implements QueryEventHandler +public class QueryScriber implements QueryEventHandler { private static final String SCRIBE_CATEGORY = "test_presto_query_complete"; private static final int MAX_QUEUE_SIZE = 1000; @@ -66,7 +66,7 @@ public QueryScriber() } @Override - public void handle(com.facebook.presto.event.query.QueryCompletionEvent event) + public void handle(QueryCompletionEvent event) { com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); Optional message = serializeThriftToString(thriftEvent); @@ -95,9 +95,10 @@ private Optional serializeThriftToString(TBase thriftMessage) } } - private static QueryCompletionEvent toThriftQueryCompletionEvent(com.facebook.presto.event.query.QueryCompletionEvent event) + private static com.twitter.presto.thriftjava.QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletionEvent event) { - QueryCompletionEvent thriftEvent = new QueryCompletionEvent(); + com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = + new com.twitter.presto.thriftjava.QueryCompletionEvent(); thriftEvent.query_id = event.getQueryId(); thriftEvent.transaction_id = event.getTransactionId(); From fadd0f13f702a516a67b29c82dc8880ef19fce42 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 10:08:39 -0700 Subject: [PATCH 73/83] Removing test from scribe category --- .../java/com/facebook/presto/twitter/logging/QueryScriber.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java index 306029550ed6..ad3e52fb9c21 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -34,7 +34,7 @@ */ public class QueryScriber implements QueryEventHandler { - private static final String SCRIBE_CATEGORY = "test_presto_query_complete"; + private static final String SCRIBE_CATEGORY = "presto_query_complete"; private static final int MAX_QUEUE_SIZE = 1000; private static final Logger log = Logger.get(QueryScriber.class); From 2e5c3f2ee54859b6f8d9a3a4c14c20da836175fd Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 10:25:12 -0700 Subject: [PATCH 74/83] change version to avoid merge conflicts --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index ae36b8cf3d9c..dbe9ff96aecd 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.141-tw-0.20 + 0.143 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 00cc629d364c..f97c8bba0c51 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 3652e17bd06f..428c8c649476 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 8f86dabd3e25..2d6d529d0fae 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141-tw-0.20 + 0.143 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 1c0d41086973..f939fcd80924 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 1bdeaecdcea2..b6558dd1ae17 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 4f0675eda340..fabc5079fb31 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index bdcff6cdb591..841119ce5d50 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index f54c94db8398..1afa4e2137b9 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 4b4ad310d3e1..0aa73f400de0 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 2c4cc361f6fb..e2cce02150c6 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 7bb7379f70b6..6321368c053f 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 09e13a19e68d..e327f50e7fd0 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 6e304d8b10bd..0b2e9211761b 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index cfa615c8d2dc..f6e99377daf2 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index cebd8614c068..2ec69e52029e 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 24ca263be84e..84c75bbc045b 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 4ae769912294..b9d05f4c16c9 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 31620ae9fe5a..2aa0870b3928 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index dfe5c5d08d7a..789340a2d1fc 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index b5e67298855a..1899f5fcca69 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 4722581cd3a5..35b56aa6b724 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index d710b58886f5..160e11c44e72 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 830af643de31..a9c126945d7a 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index c8e057ed14aa..018204dea0e0 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index bfe2383a567f..4b00570b97cd 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141-tw-0.20 + 0.143 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 3b527cbaa5dc..dee555abf4bb 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 5c724e169c0c..e8b911e4b433 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 4c226d7edc39..8c618e94b57c 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 14038c648f0f..d4a6474c05bd 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 2c4b2c1f49eb..2a35482c59a9 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index a977f2d08420..39bbf912871a 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 301e4c4efe55..bb6947fe3db6 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index dc8b11426992..a1d92c2c2639 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 25c2414ce83f..a81e47fb1e3e 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141-tw-0.20 + 0.143 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 3979e5a18555..62667b35f11d 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 9dbeee8b8252..21c117a922ad 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-verifier From 1673479f824df2bfe0c63c91233eb1709d08dad7 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 10:32:32 -0700 Subject: [PATCH 75/83] Update version to 0.143-tw-21 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index a15ff943598a..4d3e280a8b86 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.143 + 0.143-tw-21 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index f97c8bba0c51..9a3b30321ab3 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 428c8c649476..3e9996b54c4a 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 2d6d529d0fae..f678a84674f6 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143 + 0.143-tw-21 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index f939fcd80924..96374f2176cf 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index b6558dd1ae17..bb5f42df5816 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index d75efa6ae5d0..2d0b7b238afd 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 841119ce5d50..b1b1c9af3a69 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 1afa4e2137b9..89ffee3b5137 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0aa73f400de0..46f1cd60f31f 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index e2cce02150c6..ba4b98f37e6a 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index d6249e4c7649..dfab48e8c1fe 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 45dafbefb8e4..a7ed3e068632 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 5dd2f1cdb7b1..d1888fde6fe7 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 24e1bd0aa0d1..3f6eae9a0306 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 0ad2ec1f4a09..f1551983ce7c 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 84c75bbc045b..5840a8d8c5db 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index b9d05f4c16c9..e29367f1e07d 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 2aa0870b3928..b19205980eee 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 789340a2d1fc..955f03a0ec5b 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 1899f5fcca69..04c5a023509f 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 35b56aa6b724..9aecc0f8e037 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 160e11c44e72..b132a24e22e8 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a9c126945d7a..2af8ad644181 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 018204dea0e0..0929844e2231 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 6a0ad32d8c94..7f8944a0ecbd 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143 + 0.143-tw-21 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index dee555abf4bb..067fef0d7288 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index e8b911e4b433..b9cb797cbbc1 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 8c618e94b57c..10149cb92d1b 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index d4a6474c05bd..38e7af2f19cc 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 2a35482c59a9..9183920dfa3c 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 39bbf912871a..c6f3399b8ff7 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index bb6947fe3db6..81dc9466c696 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index a1d92c2c2639..ea738c49db14 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index a81e47fb1e3e..4376173271e8 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143 + 0.143-tw-21 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 62667b35f11d..9a704e9671b1 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 21c117a922ad..3511883027ed 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-verifier From 4e48cbd905adb085c1986050632beb2679796c63 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 11:07:23 -0700 Subject: [PATCH 76/83] Fixed to not use illegal import --- .../presto/twitter/hive/TestZookeeperMetastoreMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java index d0eba9f4e6e5..1f89464aedce 100644 --- a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.twitter.hive; -import com.facebook.presto.hadoop.shaded.com.google.common.collect.ImmutableList; import com.facebook.presto.twitter.hive.util.TestUtils; +import com.google.common.collect.ImmutableList; import com.google.common.net.HostAndPort; import io.airlift.log.Logger; import org.I0Itec.zkclient.ZkClient; From 55a39606d49d3706b98dc1fa20285dec76614b7c Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 11:22:11 -0700 Subject: [PATCH 77/83] Reduce logging of unsupported HttpRequest events --- .../src/main/java/com/facebook/presto/event/EventProcessor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java index c575a6a49818..eef91027a4a1 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java @@ -72,8 +72,6 @@ protected void postEvent(T event) case SPLIT_COMPLETION: handle(splitCompletionEventHandlers, type, (SplitCompletionEvent) event); break; - default: - log.warn("Unrecognized event found: " + type); } } From 8729a4ef5268876deef4c1ebc36630ddb9607fe7 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 13:17:31 -0700 Subject: [PATCH 78/83] Fix version from 0.143-tw-21 to 0.143-tw-0.21 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 4d3e280a8b86..dcd36597ae72 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.143-tw-21 + 0.143-tw-0.21 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 9a3b30321ab3..f5509ae7ace0 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 3e9996b54c4a..f4798cbed1db 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index f678a84674f6..9cdb2ba7a288 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-21 + 0.143-tw-0.21 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 96374f2176cf..cee636f28457 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index bb5f42df5816..3f9d4cc315b8 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 2d0b7b238afd..1584fab21a1a 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index b1b1c9af3a69..0112a2ec8df3 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 89ffee3b5137..86694f5e8351 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 46f1cd60f31f..802a03d546d3 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index ba4b98f37e6a..93ecc58122af 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index dfab48e8c1fe..98bf09737d7f 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index a7ed3e068632..5a5325b78ffd 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index d1888fde6fe7..c497b8c804e2 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 3f6eae9a0306..08a42164f5a9 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f1551983ce7c..f2b6bfe18024 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 5840a8d8c5db..0649fdbca865 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index e29367f1e07d..6e482242ae9e 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index b19205980eee..5bb58fec8ffc 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 955f03a0ec5b..51a6e06ebe65 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 04c5a023509f..c017ed0e635d 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 9aecc0f8e037..a85b0b403e1f 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index b132a24e22e8..a67f058a17ed 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 2af8ad644181..97e0abd530f1 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 0929844e2231..7b2409dcb2ea 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 7f8944a0ecbd..b14447d4255d 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-21 + 0.143-tw-0.21 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 067fef0d7288..98098e7bca6b 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index b9cb797cbbc1..007a962d5113 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 10149cb92d1b..d0e5fcb1a355 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 38e7af2f19cc..8c699831a585 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 9183920dfa3c..0c4200ec3f96 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index c6f3399b8ff7..26814c291cf9 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 81dc9466c696..9d0629fca8e8 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index ea738c49db14..fca9e2640ba5 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 4376173271e8..4ab85400cfe8 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-21 + 0.143-tw-0.21 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 9a704e9671b1..38342b08d9b7 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 3511883027ed..2d075a28a214 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-verifier From e6579abad2f31e5966c5edffda2ca19ed3deb5a2 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 22:40:07 -0700 Subject: [PATCH 79/83] Fix name of log category --- .../java/com/facebook/presto/twitter/logging/QueryScriber.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java index ad3e52fb9c21..5e55aa1ac195 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -34,7 +34,7 @@ */ public class QueryScriber implements QueryEventHandler { - private static final String SCRIBE_CATEGORY = "presto_query_complete"; + private static final String SCRIBE_CATEGORY = "presto_query_completion"; private static final int MAX_QUEUE_SIZE = 1000; private static final Logger log = Logger.get(QueryScriber.class); From 9681aa1372c46cd2275d275a91f6d70cdfc9135d Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Mon, 4 Apr 2016 15:53:46 -0700 Subject: [PATCH 80/83] Support adding fields to nested Parquet structs --- .../presto/hive/HiveSplitManager.java | 87 +++++++++++++------ .../hive/parquet/ParquetHiveRecordCursor.java | 6 +- .../presto/hive/AbstractTestHiveClient.java | 19 +++- .../src/test/sql/create-test-hive13.sql | 9 +- 4 files changed, 88 insertions(+), 33 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index 5aea4fe9ab87..f76a974e11c4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -69,6 +69,7 @@ public class HiveSplitManager implements ConnectorSplitManager { public static final String PRESTO_OFFLINE = "presto_offline"; + private static final String STRUCT_PREFIX = "struct<"; private final String connectorId; private final HiveMetastore metastore; @@ -220,7 +221,7 @@ private Iterable getPartitionMetadata(Table table, Schema throw new PrestoException(INTERNAL_ERROR, "Partition not loaded: " + hivePartition); } - // verify all partition is online + // verify all partitions are online String protectMode = partition.getParameters().get(ProtectMode.PARAMETER_NAME); String partName = createPartitionName(partition, table); if (protectMode != null && getProtectModeFromString(protectMode).offline) { @@ -231,32 +232,8 @@ private Iterable getPartitionMetadata(Table table, Schema throw new PartitionOfflineException(tableName, partName, format("Partition '%s' is offline for Presto: %s", partName, prestoOffline)); } - // Verify that the partition schema matches the table schema. - // Either adding or dropping columns from the end of the table - // without modifying existing partitions is allowed, but every - // column that exists in both the table and partition must have - // the same type. - List tableColumns = table.getSd().getCols(); - List partitionColumns = partition.getSd().getCols(); - if ((tableColumns == null) || (partitionColumns == null)) { - throw new PrestoException(HIVE_INVALID_METADATA, format("Table '%s' or partition '%s' has null columns", tableName, partName)); - } - for (int i = 0; i < min(partitionColumns.size(), tableColumns.size()); i++) { - String tableType = tableColumns.get(i).getType(); - String partitionType = partitionColumns.get(i).getType(); - if (!tableType.equals(partitionType)) { - throw new PrestoException(HIVE_PARTITION_SCHEMA_MISMATCH, format("" + - "There is a mismatch between the table and partition schemas. " + - "The column '%s' in table '%s' is declared as type '%s', " + - "but partition '%s' declared column '%s' as type '%s'.", - tableColumns.get(i).getName(), - tableName, - tableType, - partName, - partitionColumns.get(i).getName(), - partitionType)); - } - } + verifySchemaEvolution(tableName, table.getSd().getCols(), + partName, table.getSd().getCols()); results.add(new HivePartitionMetadata(hivePartition, partition)); } @@ -266,6 +243,62 @@ private Iterable getPartitionMetadata(Table table, Schema return concat(partitionBatches); } + /** + * Verify that the partition schema is backward compatible with the table schema. Either + * adding or dropping columns from the end of the table without modifying existing partitions is + * allowed, but every column that exists in both the table and partition must have the same type. + * + * If the type is a nested type (i.e. a struct), verify all partition struct fields exist in the + * table struct fields in with the same name, type and order. Af that condition is met, additional + * table struct fields are permitted. + * + * @throws PrestoException if the schema has not evolved in a supported way + */ + private void verifySchemaEvolution(SchemaTableName tableName, List tableColumns, + String partitionName, List partitionColumns) throws PrestoException + { + if ((tableColumns == null) || (partitionColumns == null)) { + throw new PrestoException(HIVE_INVALID_METADATA, format("Table '%s' or partition '%s' has null columns", tableName, partitionName)); + } + for (int i = 0; i < min(partitionColumns.size(), tableColumns.size()); i++) { + String tableType = tableColumns.get(i).getType(); + String partitionType = partitionColumns.get(i).getType(); + boolean validEvolution; + if (isStruct(tableType) && isStruct(partitionType)) { + validEvolution = structFields(tableType).startsWith(structFields(partitionType)); + } + else { + validEvolution = tableType.equals(partitionType); + } + + if (!validEvolution) { + throw new PrestoException(HIVE_PARTITION_SCHEMA_MISMATCH, format("" + + "There is a mismatch between the table and partition schemas. " + + "The column '%s' in table '%s' is declared as type '%s', " + + "but partition '%s' declared column '%s' as type '%s'.", + tableColumns.get(i).getName(), + tableName, + tableType, + partitionName, + partitionColumns.get(i).getName(), + partitionType)); + } + } + } + + // Struct types take the following form: + // struct + private static boolean isStruct(String type) + { + return type.startsWith(STRUCT_PREFIX); + } + + // remove leading 'struct<' and trailing '>' + private static String structFields(String structType) + { + return structType.substring(STRUCT_PREFIX.length(), structType.length() - 1); + } + /** * Partition the given list in exponentially (power of 2) increasing batch sizes starting at 1 up to maxBatchSize */ diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java index 46b06da792de..c47d4948416f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java @@ -729,7 +729,7 @@ public ParquetStructConverter(Type prestoType, String columnName, GroupType entr List prestoTypeParameters = prestoType.getTypeParameters(); List fieldTypes = entryType.getFields(); checkArgument( - prestoTypeParameters.size() == fieldTypes.size(), + prestoTypeParameters.size() >= fieldTypes.size(), "Schema mismatch, metastore schema for row column %s has %s fields but parquet schema has %s fields", columnName, prestoTypeParameters.size(), @@ -739,7 +739,7 @@ public ParquetStructConverter(Type prestoType, String columnName, GroupType entr this.fieldIndex = fieldIndex; ImmutableList.Builder converters = ImmutableList.builder(); - for (int i = 0; i < prestoTypeParameters.size(); i++) { + for (int i = 0; i < fieldTypes.size(); i++) { parquet.schema.Type fieldType = fieldTypes.get(i); converters.add(createConverter(prestoTypeParameters.get(i), columnName + "." + fieldType.getName(), fieldType, i)); } @@ -784,7 +784,7 @@ public void end() for (BlockConverter converter : converters) { converter.afterValue(); } - while (currentEntryBuilder.getPositionCount() < converters.size()) { + while (currentEntryBuilder.getPositionCount() < rowType.getTypeParameters().size()) { currentEntryBuilder.appendNull(); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index c4cf4f91c9d3..14ff33018b80 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -67,6 +67,7 @@ import com.facebook.presto.testing.TestingConnectorSession; import com.facebook.presto.type.ArrayType; import com.facebook.presto.type.MapType; +import com.facebook.presto.type.RowType; import com.facebook.presto.type.TypeRegistry; import com.facebook.presto.util.ImmutableCollectors; import com.google.common.collect.ImmutableList; @@ -2273,6 +2274,22 @@ protected void assertGetRecords( } } + // STRUCT + index = columnIndex.get("t_struct"); + if (index != null) { + if ((rowNumber % 31) == 0) { + assertNull(row.getField(index)); + } + else { + assertTrue(row.getField(index) instanceof List); + List values = (List) row.getField(index); + assertEquals(values.size(), 3); + assertEquals(values.get(0), "test abc"); + assertEquals(values.get(1), 0.1); + assertNull(values.get(2)); + } + } + // MAP>> index = columnIndex.get("t_complex"); if (index != null) { @@ -2482,7 +2499,7 @@ else if (TIMESTAMP.equals(column.getType())) { else if (DATE.equals(column.getType())) { assertInstanceOf(value, SqlDate.class); } - else if (column.getType() instanceof ArrayType) { + else if (column.getType() instanceof ArrayType || column.getType() instanceof RowType) { assertInstanceOf(value, List.class); } else if (column.getType() instanceof MapType) { diff --git a/presto-hive/src/test/sql/create-test-hive13.sql b/presto-hive/src/test/sql/create-test-hive13.sql index 70c1d349d1e1..29235f1ba996 100644 --- a/presto-hive/src/test/sql/create-test-hive13.sql +++ b/presto-hive/src/test/sql/create-test-hive13.sql @@ -82,7 +82,6 @@ SELECT * FROM presto_test_types_textfile ; --- Parquet fails when trying to use complex nested types. CREATE TABLE presto_test_types_parquet ( t_string STRING , t_tinyint TINYINT @@ -96,12 +95,15 @@ CREATE TABLE presto_test_types_parquet ( , t_binary BINARY , t_map MAP , t_array_string ARRAY -, t_array_struct ARRAY> +, t_array_struct ARRAY> +, t_struct STRUCT ) +PARTITIONED BY (dummy INT) STORED AS PARQUET ; INSERT INTO TABLE presto_test_types_parquet +PARTITION (dummy=0) SELECT t_string , t_tinyint @@ -116,9 +118,12 @@ SELECT , t_map , t_array_string , t_array_struct +, t_array_struct[0] FROM presto_test_types_textfile ; +ALTER TABLE presto_test_types_parquet +CHANGE COLUMN t_struct t_struct STRUCT; ALTER TABLE presto_test_types_textfile ADD COLUMNS (new_column INT); ALTER TABLE presto_test_types_sequencefile ADD COLUMNS (new_column INT); From b2cea9fdfd166efe8ec0e600c1b8fdb90235baa7 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Mon, 4 Apr 2016 21:57:56 -0700 Subject: [PATCH 81/83] Fix table partition typo --- .../main/java/com/facebook/presto/hive/HiveSplitManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index f76a974e11c4..fc7cc22aacaa 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -233,7 +233,7 @@ private Iterable getPartitionMetadata(Table table, Schema } verifySchemaEvolution(tableName, table.getSd().getCols(), - partName, table.getSd().getCols()); + partName, partition.getSd().getCols()); results.add(new HivePartitionMetadata(hivePartition, partition)); } From f85de5300a07dbcdb8cf53d3005263478c84c55a Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 5 Apr 2016 11:15:20 -0700 Subject: [PATCH 82/83] Fix table partition typo --- .../main/java/com/facebook/presto/hive/HiveSplitManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index fc7cc22aacaa..7fd56a874b11 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -233,7 +233,7 @@ private Iterable getPartitionMetadata(Table table, Schema } verifySchemaEvolution(tableName, table.getSd().getCols(), - partName, partition.getSd().getCols()); + partName, partition.getSd().getCols()); results.add(new HivePartitionMetadata(hivePartition, partition)); } From 9b936c8d10eb4c0e98880c2a497b7767391c54b4 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 5 Apr 2016 12:00:51 -0700 Subject: [PATCH 83/83] Refactored type comparison to use HiveType --- .../presto/hive/HiveSplitManager.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index 7fd56a874b11..b545ef61e754 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -35,9 +35,13 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import javax.inject.Inject; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -69,7 +73,6 @@ public class HiveSplitManager implements ConnectorSplitManager { public static final String PRESTO_OFFLINE = "presto_offline"; - private static final String STRUCT_PREFIX = "struct<"; private final String connectorId; private final HiveMetastore metastore; @@ -261,11 +264,13 @@ private void verifySchemaEvolution(SchemaTableName tableName, List throw new PrestoException(HIVE_INVALID_METADATA, format("Table '%s' or partition '%s' has null columns", tableName, partitionName)); } for (int i = 0; i < min(partitionColumns.size(), tableColumns.size()); i++) { - String tableType = tableColumns.get(i).getType(); - String partitionType = partitionColumns.get(i).getType(); + HiveType tableType = HiveType.valueOf(tableColumns.get(i).getType()); + HiveType partitionType = HiveType.valueOf(partitionColumns.get(i).getType()); boolean validEvolution; if (isStruct(tableType) && isStruct(partitionType)) { - validEvolution = structFields(tableType).startsWith(structFields(partitionType)); + ArrayList tableFieldTypes = getStructFields(tableType); + ArrayList partitionFieldTypes = getStructFields(partitionType); + validEvolution = tableFieldTypes.subList(0, partitionFieldTypes.size()).equals(partitionFieldTypes); } else { validEvolution = tableType.equals(partitionType); @@ -286,17 +291,14 @@ private void verifySchemaEvolution(SchemaTableName tableName, List } } - // Struct types take the following form: - // struct - private static boolean isStruct(String type) + private static boolean isStruct(HiveType type) { - return type.startsWith(STRUCT_PREFIX); + return type.getCategory() == Category.STRUCT; } - // remove leading 'struct<' and trailing '>' - private static String structFields(String structType) + private static ArrayList getStructFields(HiveType structHiveType) { - return structType.substring(STRUCT_PREFIX.length(), structType.length() - 1); + return ((StructTypeInfo) structHiveType.getTypeInfo()).getAllStructFieldTypeInfos(); } /**