Support adding fields to nested Parquet structs

pom.xml

@@ -10,7 +10,7 @@
 
     <groupId>com.facebook.presto</groupId>
     <artifactId>presto-root</artifactId>
-    <version>0.143</version>
+    <version>0.143-tw-0.21</version>
     <packaging>pom</packaging>
 
     <name>presto-root</name>
@@ -28,9 +28,9 @@
     </licenses>
 
     <scm>
-        <connection>scm:git:git://github.com/facebook/presto.git</connection>
-        <url>https://github.com/facebook/presto</url>
-        <tag>0.143</tag>
+        <connection>scm:git:git://github.com/twitter-forks/presto.git</connection>
+        <url>https://github.com/twitter-forks/presto</url>
+        <tag>0.143-tw-0.21</tag>
     </scm>
 
     <properties>
@@ -663,7 +663,7 @@
             <dependency>
                 <groupId>org.apache.zookeeper</groupId>
                 <artifactId>zookeeper</artifactId>
-                <version>3.3.6</version>
+                <version>3.4.6</version>
                 <exclusions>
                     <exclusion>
                         <artifactId>junit</artifactId>
@@ -673,6 +673,14 @@
                         <artifactId>log4j</artifactId>
                         <groupId>log4j</groupId>
                     </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-jdk14</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-log4j12</artifactId>
+                    </exclusion>
                 </exclusions>
             </dependency>
 
@@ -729,6 +737,61 @@
                 <artifactId>hive-apache-jdbc</artifactId>
                 <version>0.13.1-1</version>
             </dependency>
+
+            <!-- twitter deps -->
+            <dependency>
+                <groupId>com.twitter</groupId>
+                <artifactId>presto-thrift-java</artifactId>
+                <version>0.0.1</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.twitter</groupId>
+                        <artifactId>util-core_2.11</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>com.twitter</groupId>
+                        <artifactId>util-core-java</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>com.twitter</groupId>
+                        <artifactId>util-function_2.10</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>com.twitter</groupId>
+                        <artifactId>util-function-java</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.scala-lang.modules</groupId>
+                        <artifactId>scala-parser-combinators_2.11</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>com.twitter</groupId>
+                <artifactId>util-logging_2.11</artifactId>
+                <version>6.33.0</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>org.scala-lang</groupId>
+                <artifactId>scala-library</artifactId>
+                <version>2.11.7</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 

presto-base-jdbc/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-base-jdbc</artifactId>

presto-benchmark-driver/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-benchmark-driver</artifactId>

presto-benchmark/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>presto-root</artifactId>
         <groupId>com.facebook.presto</groupId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-benchmark</artifactId>

presto-blackhole/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-blackhole</artifactId>

presto-bytecode/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-bytecode</artifactId>

presto-cassandra/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-cassandra</artifactId>

presto-cli/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-cli</artifactId>

presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java

@@ -17,6 +17,7 @@
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.net.HostAndPort;
+import com.sun.security.auth.module.UnixSystem;
 import io.airlift.airline.Option;
 import io.airlift.http.client.spnego.KerberosConfig;
 import io.airlift.units.Duration;
@@ -68,8 +69,9 @@ public class ClientOptions
     @Option(name = "--keystore-password", title = "keystore password", description = "Keystore password")
     public String keystorePassword;
 
-    @Option(name = "--user", title = "user", description = "Username")
-    public String user = System.getProperty("user.name");
+    // Pick the user name for the logged in user.
+    // Do not let it be overridden by users.
+    public String user = new UnixSystem().getUsername();
 
     @Option(name = "--source", title = "source", description = "Name of source making query")
     public String source = "presto-cli";

presto-client/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-client</artifactId>

presto-docs/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-docs</artifactId>

presto-example-http/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-example-http</artifactId>

presto-hive-cdh4/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-hive-cdh4</artifactId>

presto-hive-cdh5/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-hive-cdh5</artifactId>

presto-hive-hadoop1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-hive-hadoop1</artifactId>

presto-hive-hadoop2/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-hive-hadoop2</artifactId>

presto-hive/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.143</version>
+        <version>0.143-tw-0.21</version>
     </parent>
 
     <artifactId>presto-hive</artifactId>
@@ -43,6 +43,44 @@
             <artifactId>hive-apache</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-recipes</artifactId>
+            <version>2.8.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-framework</artifactId>
+            <version>2.8.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-client</artifactId>
+            <version>2.8.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-test</artifactId>
+            <version>2.8.0</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper</artifactId>
+            <version>3.4.6</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.101tec</groupId>
+            <artifactId>zkclient</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.thrift</groupId>
             <artifactId>libthrift</artifactId>
@@ -88,6 +126,12 @@
             <artifactId>configuration</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.googlecode.json-simple</groupId>
+            <artifactId>json-simple</artifactId>
+            <version>1.1</version>
+        </dependency>
+
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>

presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java

@@ -19,6 +19,7 @@
 import com.facebook.presto.spi.ConnectorSession;
 import com.facebook.presto.spi.HostAddress;
 import com.facebook.presto.spi.predicate.TupleDomain;
+import com.facebook.presto.twitter.hive.util.UgiUtils;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import io.airlift.units.DataSize;
@@ -38,8 +39,10 @@
 import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.security.UserGroupInformation;
 
 import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Deque;
@@ -138,8 +141,15 @@ public BackgroundHiveSplitLoader(
     public void start(HiveSplitSource splitSource)
     {
         this.hiveSplitSource = splitSource;
+
+        UserGroupInformation ugi = null;
+
+        if (HiveSessionProperties.getReadAsQueryUser(session)) {
+            ugi = UgiUtils.getUgi(session.getUser());
+        }
+
         for (int i = 0; i < maxPartitionBatchSize; i++) {
-            ResumableTasks.submit(executor, new HiveSplitLoaderTask());
+            ResumableTasks.submit(executor, new HiveSplitLoaderTask(ugi));
         }
     }
 
@@ -152,8 +162,30 @@ public void stop()
     private class HiveSplitLoaderTask
             implements ResumableTask
     {
+        private UserGroupInformation ugi;
+
+        public HiveSplitLoaderTask(UserGroupInformation ugi)
+        {
+            this.ugi = ugi;
+        }
+
         @Override
         public TaskStatus process()
+        {
+            if (ugi != null) {
+                try {
+                    return ugi.doAs((PrivilegedExceptionAction<TaskStatus>) this::doProcess);
+                }
+                catch (IOException | InterruptedException e) {
+                    throw new RuntimeException("Could not runAs " + session.getUser(), e);
+                }
+            }
+            else {
+                return doProcess();
+            }
+        }
+
+        private TaskStatus doProcess()
         {
             while (true) {
                 if (stopped) {