Merge branch 'main' of https://github.com/acvictor/velox into acvicto…

…r/weekday
acvictor · Feb 15, 2024 · 3d657ea · 3d657ea
2 parents 8ba9efe + 2068b95
commit 3d657ea
Show file tree

Hide file tree

Showing 47 changed files with 1,693 additions and 388 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -62,6 +62,11 @@ option(
   VELOX_BUILD_MINIMAL
   "Build a minimal set of components only. This will override other build options."
   OFF)
+option(
+  VELOX_BUILD_MINIMAL_WITH_DWIO
+  "Build a minimal set of components, including DWIO (file format readers/writers).
+   This will override other build options."
+  OFF)
 
 # option() always creates a BOOL variable so we have to use a normal cache
 # variable with STRING type for this option.
@@ -96,8 +101,10 @@ option(VELOX_ENABLE_PARQUET "Enable Parquet support" OFF)
 option(VELOX_ENABLE_ARROW "Enable Arrow support" OFF)
 option(VELOX_ENABLE_REMOTE_FUNCTIONS "Enable remote function support" OFF)
 option(VELOX_ENABLE_CCACHE "Use ccache if installed." ON)
+option(VELOX_ENABLE_CODEGEN_SUPPORT "Enable experimental codegen support." OFF)
 
 option(VELOX_BUILD_TEST_UTILS "Builds Velox test utilities" OFF)
+option(VELOX_BUILD_VECTOR_TEST_UTILS "Builds Velox vector test utilities" OFF)
 option(VELOX_BUILD_PYTHON_PACKAGE "Builds Velox Python bindings" OFF)
 option(
   VELOX_ENABLE_INT64_BUILD_PARTITION_BOUND
@@ -125,7 +132,7 @@ if(${VELOX_BUILD_MINIMAL})
   set(VELOX_ENABLE_GCS OFF)
   set(VELOX_ENABLE_ABFS OFF)
   set(VELOX_ENABLE_SUBSTRAIT OFF)
-  set(VELOX_CODEGEN_SUPPORT OFF)
+  set(VELOX_ENABLE_CODEGEN_SUPPORT OFF)
 endif()
 
 if(${VELOX_BUILD_TESTING})
@@ -175,7 +182,7 @@ if(${VELOX_BUILD_PYTHON_PACKAGE})
   set(VELOX_ENABLE_GCS OFF)
   set(VELOX_ENABLE_ABFS OFF)
   set(VELOX_ENABLE_SUBSTRAIT OFF)
-  set(VELOX_CODEGEN_SUPPORT OFF)
+  set(VELOX_ENABLE_CODEGEN_SUPPORT OFF)
   set(VELOX_ENABLE_BENCHMARKS_BASIC OFF)
   set(VELOX_ENABLE_BENCHMARKS OFF)
 endif()
@@ -257,7 +264,7 @@ if(VELOX_ENABLE_PARQUET)
 endif()
 
 # define processor variable for conditional compilation
-if(${VELOX_CODEGEN_SUPPORT})
+if(${VELOX_ENABLE_CODEGEN_SUPPORT})
   add_compile_definitions(CODEGEN_ENABLED=1)
 endif()
 
@@ -420,7 +427,10 @@ endif()
 set_source(fmt)
 resolve_dependency(fmt 9.0.0)
 
-if(NOT ${VELOX_BUILD_MINIMAL})
+if(${VELOX_BUILD_MINIMAL_WITH_DWIO} OR ${VELOX_ENABLE_HIVE_CONNECTOR})
+  # DWIO needs all sorts of stream compression libraries.
+  #
+  # TODO: make these optional and pluggable.
   find_package(ZLIB REQUIRED)
   find_package(lz4 REQUIRED)
   find_package(lzo2 REQUIRED)
@@ -467,7 +477,11 @@ else()
   set(FOLLY_BENCHMARK Folly::follybenchmark)
 endif()
 
-if(NOT ${VELOX_BUILD_MINIMAL})
+# DWIO (ORC/DWRF), Substrait and experimental/codegen depend on protobuf.
+if(${VELOX_BUILD_MINIMAL_WITH_DWIO}
+   OR ${VELOX_ENABLE_HIVE_CONNECTOR}
+   OR ${VELOX_ENABLE_SUBSTRAIT}
+   OR ${VELOX_ENABLE_CODEGEN_SUPPORT})
   # Locate or build protobuf.
   set_source(Protobuf)
   resolve_dependency(Protobuf 3.21 EXACT)

diff --git a/Makefile b/Makefile
@@ -98,10 +98,26 @@ release:				#: Build the release version
 	$(MAKE) cmake BUILD_DIR=release BUILD_TYPE=Release && \
 	$(MAKE) build BUILD_DIR=release
 
-min_debug:				#: Minimal build with debugging symbols
+minimal_debug:			#: Minimal build with debugging symbols
 	$(MAKE) cmake BUILD_DIR=debug BUILD_TYPE=debug EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DVELOX_BUILD_MINIMAL=ON"
 	$(MAKE) build BUILD_DIR=debug
 
+min_debug: minimal_debug
+
+minimal:				 #: Minimal build
+	$(MAKE) cmake BUILD_DIR=release BUILD_TYPE=release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DVELOX_BUILD_MINIMAL=ON"
+	$(MAKE) build BUILD_DIR=release
+
+dwio:						#: Minimal build with dwio enabled.
+	$(MAKE) cmake BUILD_DIR=release BUILD_TYPE=release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} \
+																										    							  -DVELOX_BUILD_MINIMAL_WITH_DWIO=ON"
+	$(MAKE) build BUILD_DIR=release
+
+dwio_debug:			#: Minimal build with dwio debugging symbols.
+	$(MAKE) cmake BUILD_DIR=debug BUILD_TYPE=debug EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} \
+																																	  -DVELOX_BUILD_MINIMAL_WITH_DWIO=ON"
+	$(MAKE) build BUILD_DIR=debug
+
 benchmarks-basic-build:
 	$(MAKE) release EXTRA_CMAKE_FLAGS=" ${EXTRA_CMAKE_FLAGS} \
                                             -DVELOX_BUILD_TESTING=OFF \

diff --git a/scripts/setup-centos8.sh b/scripts/setup-centos8.sh
@@ -30,103 +30,173 @@ function dnf_install {
   dnf install -y -q --setopt=install_weak_deps=False "$@"
 }
 
+dnf update -y
 dnf_install epel-release dnf-plugins-core # For ccache, ninja
 dnf config-manager --set-enabled powertools
+dnf update -y
 dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which libevent-devel \
   openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
   libdwarf-devel curl-devel libicu-devel
 
-dnf remove -y gflags
-
-# Required for Thrift
 dnf_install autoconf automake libtool bison flex python3 libsodium-devel
 
-dnf_install conda
-
 # install sphinx for doc gen
 pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme
 
 # Activate gcc9; enable errors on unset variables afterwards.
 source /opt/rh/gcc-toolset-9/enable || exit 1
 set -u
 
-function cmake_install {
-  cmake -B "$1-build" -GNinja -DCMAKE_CXX_STANDARD=17 \
-    -DCMAKE_CXX_FLAGS="${CFLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" -Wno-dev "$@"
-  ninja -C "$1-build" install
+function install_conda {
+  dnf_install conda
 }
 
-# Fetch sources.
-wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
-wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
-wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo
-wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost
-wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
-wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
+function install_gflags {
+  # Remove an older version if present.
+  dnf remove -y gflags
+  wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
+  (
+    cd gflags
+    cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
+  )
+}
 
-wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz protobuf
+function install_glog {
+  wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
+  (
+    cd glog
+    cmake_install -DBUILD_SHARED_LIBS=ON
+  )
+}
+
+function install_lzo {
+  wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo
+  (
+    cd lzo
+    ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10
+    make "-j$(nproc)"
+    make install
+  )
+}
+
+function install_boost {
+  wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost
+  (
+   cd boost
+   ./bootstrap.sh --prefix=/usr/local
+   ./b2 "-j$(nproc)" -d0 install threading=multi
+  )
+}
+
+function install_snappy {
+  wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
+  (
+    cd snappy
+    cmake_install -DSNAPPY_BUILD_TESTS=OFF
+  )
+}
+
+function install_fmt {
+  wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
+  (
+    cd fmt
+    cmake_install -DFMT_TEST=OFF
+  )
+}
+
+function install_protobuf {
+  wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz protobuf
+  (
+    cd protobuf
+    ./configure --prefix=/usr
+    make "-j${NPROC}"
+    make install
+    ldconfig
+  )
+}
 
 FB_OS_VERSION="v2023.12.04.00"
 
-wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
-wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
-wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
-wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
-wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
+function install_fizz {
+  wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
+  (
+    cd fizz/fizz
+    cmake_install -DBUILD_TESTS=OFF
+  )
+}
 
-wait  # For cmake and source downloads to complete.
+function install_folly {
+  wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
+  (
+    cd folly
+    cmake_install -DFOLLY_HAVE_INT128_T=ON
+  )
+}
 
-# Build & install.
-(
-  cd lzo
-  ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10
-  make "-j$(nproc)"
-  make install
-)
+function install_wangle {
+  wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
+  (
+    cd wangle/wangle
+    cmake_install -DBUILD_TESTS=OFF
+  )
+}
 
-(
-  cd boost
-  ./bootstrap.sh --prefix=/usr/local
-  ./b2 "-j$(nproc)" -d0 install threading=multi
-)
+function install_fbthrift {
+  wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
+  (
+    cd fbthrift
+    cmake_install -Denable_tests=OFF
+  )
+}
+
+function install_mvfst {
+  wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
+  (
+   cd mvfst
+   cmake_install -DBUILD_TESTS=OFF
+  )
+}
+
+function install_duckdb {
+  if $BUILD_DUCKDB ; then
+    echo 'Building DuckDB'
+    wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
+    (
+      cd duckdb
+      cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
+    )
+  fi
+}
+
+function install_velox_deps {
+  run_and_time install_conda
+  run_and_time install_gflags
+  run_and_time install_glog
+  run_and_time install_lzo
+  run_and_time install_snappy
+  run_and_time install_boost
+  run_and_time install_protobuf
+  run_and_time install_fmt
+  run_and_time install_folly
+  run_and_time install_fizz
+  run_and_time install_wangle
+  run_and_time install_mvfst
+  run_and_time install_fbthrift
+  run_and_time install_duckdb
+}
+
+(return 2> /dev/null) && return # If script was sourced, don't run commands.
 
 (
-  cd protobuf
-  ./configure --prefix=/usr
-  make "-j${NPROC}"
-  make install
-  ldconfig
+  if [[ $# -ne 0 ]]; then
+    for cmd in "$@"; do
+      run_and_time "${cmd}"
+    done
+  else
+    install_velox_deps
+  fi
 )
 
-cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr
-cmake_install glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr
-cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
-cmake_install fmt -DFMT_TEST=OFF
-cmake_install folly -DFOLLY_HAVE_INT128_T=ON
-
-cmake_install fizz/fizz -DBUILD_TESTS=OFF
-cmake_install wangle/wangle -DBUILD_TESTS=OFF
-cmake_install mvfst -DBUILD_TESTS=OFF
-cmake_install fbthrift -Denable_tests=OFF
-
-if $BUILD_DUCKDB ; then
-  echo 'Building DuckDB'
-  mkdir ~/duckdb-install && cd ~/duckdb-install
-  wget https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz
-  tar -xf v0.8.1.tar.gz
-  cd duckdb-0.8.1
-  mkdir build && cd build
-  CMAKE_FLAGS=(
-    "-DBUILD_UNITTESTS=OFF"
-    "-DENABLE_SANITIZER=OFF"
-    "-DENABLE_UBSAN=OFF"
-    "-DBUILD_SHELL=OFF"
-    "-DEXPORT_DLL_SYMBOLS=OFF"
-    "-DCMAKE_BUILD_TYPE=Release"
-  )
-  cmake ${CMAKE_FLAGS[*]}  ..
-  make install -j 16
-  rm -rf ~/duckdb-install
-fi
+echo "All dependencies for Velox installed!"
 
 dnf clean all
diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh
@@ -15,6 +15,27 @@
 
 # github_checkout $REPO $VERSION $GIT_CLONE_PARAMS clones or re-uses an existing clone of the
 # specified repo, checking out the requested version.
+
+function run_and_time {
+  time "$@" || (echo "Failed to run $* ." ; exit 1 )
+  { echo "+ Finished running $*"; } 2> /dev/null
+}
+
+function prompt {
+  (
+    while true; do
+      local input="${PROMPT_ALWAYS_RESPOND:-}"
+      echo -n "$(tput bold)$* [Y, n]$(tput sgr0) "
+      [[ -z "${input}" ]] && read input
+      if [[ "${input}" == "Y" || "${input}" == "y" || "${input}" == "" ]]; then
+        return 0
+      elif [[ "${input}" == "N" || "${input}" == "n" ]]; then
+        return 1
+      fi
+    done
+  ) 2> /dev/null
+}
+
 function github_checkout {
   local REPO=$1
   shift
@@ -36,7 +57,6 @@ function github_checkout {
   cd "${DIRNAME}"
 }
 
-
 # get_cxx_flags [$CPU_ARCH]
 # Sets and exports the variable VELOX_CXX_FLAGS with appropriate compiler flags.
 # If $CPU_ARCH is set then we use that else we determine best possible set of flags