Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/acvictor/velox into acvicto…
Browse files Browse the repository at this point in the history
…r/unixdate
  • Loading branch information
acvictor committed Feb 17, 2024
2 parents 31798df + 1649035 commit 6e828fa
Show file tree
Hide file tree
Showing 96 changed files with 4,266 additions and 1,073 deletions.
36 changes: 0 additions & 36 deletions .circleci/dist_compile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -431,38 +431,6 @@ jobs:
fuzzer_exe: "_build/debug/velox/exec/tests/velox_join_fuzzer_test"
fuzzer_args: " --seed ${RANDOM} --duration_sec 3600 --logtostderr=1 --minloglevel=0"

format-check:
executor: check
steps:
- checkout
- run:
name: Check formatting
command: |
if ! make format-check; then
make format-fix
echo -e "\n==== Apply using:"
echo "patch -p1 \<<EOF"
git --no-pager diff
echo "EOF"
false
fi
header-check:
executor: check
steps:
- checkout
- run:
name: Check license headers
command: |
if ! make header-check; then
make header-fix
echo -e "\n==== Apply using:"
echo "patch -p1 \<<EOF"
git --no-pager diff
echo "EOF"
false
fi
doc-gen-job:
executor: build
steps:
Expand Down Expand Up @@ -604,8 +572,6 @@ workflows:
- linux-build-options
- linux-adapters
- linux-presto-fuzzer-run
- format-check
- header-check
- doc-gen-job:
filters:
branches:
Expand All @@ -619,8 +585,6 @@ workflows:
- linux-pr-fuzzer-run
- linux-build-options
- linux-adapters
- format-check
- header-check
- doc-gen-job:
filters:
branches:
Expand Down
22 changes: 18 additions & 4 deletions .github/workflows/macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,22 @@ name: macOS Build

on:
push:
paths:
- "velox/**"
- "!velox/docs/**"
- "CMakeLists.txt"
- "CMake/**"
- "third_party/**"
- ".github/workflows/macos.yml"

pull_request:
paths:
- "velox/**"
- "!velox/docs/**"
- "CMakeLists.txt"
- "CMake/**"
- "third_party/**"
- ".github/workflows/macos.yml"

permissions:
contents: read
Expand Down Expand Up @@ -48,16 +63,16 @@ jobs:
bison boost ccache double-conversion flex fmt gflags glog \
icu4c libevent libsodium lz4 lzo ninja openssl range-v3 simdjson \
snappy thrift xz xsimd zstd
echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV
- name: Cache ccache
uses: actions/cache@v4
with:
path: '${{ env.CCACHE_DIR }}'
key: ccache-macos-${{ matrix.os }}-${{ hashFiles('velox/*') }}
restore-keys: ccache-macos-${{ matrix.os }}

- name: Configure Build
env:
folly_SOURCE: BUNDLED
Expand All @@ -78,4 +93,3 @@ jobs:
- name: Run Tests
if: false
run: ctest -j $NJOBS --test-dir _build/debug --output-on-failure

75 changes: 75 additions & 0 deletions .github/workflows/preliminary_checks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Run Checks

on:
pull_request:

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }}
cancel-in-progress: true

jobs:
check-matrix:
name: ${{ matrix.config.name }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
config:
- { name: "License Header",
command: "header-fix",
message: "Found missing License Header(s)",
reqs: "regex"
}
- { name: "Code Format",
command: "format-fix",
message: "Found format issues",
reqs: "regex cmake-format black"
}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Install Dependencies
run: |
python -m venv check_env
source check_env/bin/activate
pip install ${{ matrix.config.reqs }}
- name: Check ${{ matrix.config.name }}
run: |
source check_env/bin/activate
make ${{ matrix.config.command }}
if ! git diff --quiet; then
diff=`git --no-pager diff`
echo "${{ matrix.command.message }} in the following files:"
git --no-pager diff --name-only
echo "Check the Job summary for a copy-pasteable patch."
echo "> [!IMPORTANT]" >> $GITHUB_STEP_SUMMARY
echo "${{ matrix.config.message }}" >> $GITHUB_STEP_SUMMARY
echo "> Please apply fix using:" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`sh" >> $GITHUB_STEP_SUMMARY
echo "patch -p1 <<EOF" >> $GITHUB_STEP_SUMMARY
echo "$diff" >> $GITHUB_STEP_SUMMARY
echo "EOF" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
exit 1
fi
24 changes: 19 additions & 5 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ option(
VELOX_BUILD_MINIMAL
"Build a minimal set of components only. This will override other build options."
OFF)
option(
VELOX_BUILD_MINIMAL_WITH_DWIO
"Build a minimal set of components, including DWIO (file format readers/writers).
This will override other build options."
OFF)

# option() always creates a BOOL variable so we have to use a normal cache
# variable with STRING type for this option.
Expand Down Expand Up @@ -96,8 +101,10 @@ option(VELOX_ENABLE_PARQUET "Enable Parquet support" OFF)
option(VELOX_ENABLE_ARROW "Enable Arrow support" OFF)
option(VELOX_ENABLE_REMOTE_FUNCTIONS "Enable remote function support" OFF)
option(VELOX_ENABLE_CCACHE "Use ccache if installed." ON)
option(VELOX_ENABLE_CODEGEN_SUPPORT "Enable experimental codegen support." OFF)

option(VELOX_BUILD_TEST_UTILS "Builds Velox test utilities" OFF)
option(VELOX_BUILD_VECTOR_TEST_UTILS "Builds Velox vector test utilities" OFF)
option(VELOX_BUILD_PYTHON_PACKAGE "Builds Velox Python bindings" OFF)
option(
VELOX_ENABLE_INT64_BUILD_PARTITION_BOUND
Expand Down Expand Up @@ -125,7 +132,7 @@ if(${VELOX_BUILD_MINIMAL})
set(VELOX_ENABLE_GCS OFF)
set(VELOX_ENABLE_ABFS OFF)
set(VELOX_ENABLE_SUBSTRAIT OFF)
set(VELOX_CODEGEN_SUPPORT OFF)
set(VELOX_ENABLE_CODEGEN_SUPPORT OFF)
endif()

if(${VELOX_BUILD_TESTING})
Expand Down Expand Up @@ -175,7 +182,7 @@ if(${VELOX_BUILD_PYTHON_PACKAGE})
set(VELOX_ENABLE_GCS OFF)
set(VELOX_ENABLE_ABFS OFF)
set(VELOX_ENABLE_SUBSTRAIT OFF)
set(VELOX_CODEGEN_SUPPORT OFF)
set(VELOX_ENABLE_CODEGEN_SUPPORT OFF)
set(VELOX_ENABLE_BENCHMARKS_BASIC OFF)
set(VELOX_ENABLE_BENCHMARKS OFF)
endif()
Expand Down Expand Up @@ -257,7 +264,7 @@ if(VELOX_ENABLE_PARQUET)
endif()

# define processor variable for conditional compilation
if(${VELOX_CODEGEN_SUPPORT})
if(${VELOX_ENABLE_CODEGEN_SUPPORT})
add_compile_definitions(CODEGEN_ENABLED=1)
endif()

Expand Down Expand Up @@ -420,7 +427,10 @@ endif()
set_source(fmt)
resolve_dependency(fmt 9.0.0)

if(NOT ${VELOX_BUILD_MINIMAL})
if(${VELOX_BUILD_MINIMAL_WITH_DWIO} OR ${VELOX_ENABLE_HIVE_CONNECTOR})
# DWIO needs all sorts of stream compression libraries.
#
# TODO: make these optional and pluggable.
find_package(ZLIB REQUIRED)
find_package(lz4 REQUIRED)
find_package(lzo2 REQUIRED)
Expand Down Expand Up @@ -467,7 +477,11 @@ else()
set(FOLLY_BENCHMARK Folly::follybenchmark)
endif()

if(NOT ${VELOX_BUILD_MINIMAL})
# DWIO (ORC/DWRF), Substrait and experimental/codegen depend on protobuf.
if(${VELOX_BUILD_MINIMAL_WITH_DWIO}
OR ${VELOX_ENABLE_HIVE_CONNECTOR}
OR ${VELOX_ENABLE_SUBSTRAIT}
OR ${VELOX_ENABLE_CODEGEN_SUPPORT})
# Locate or build protobuf.
set_source(Protobuf)
resolve_dependency(Protobuf 3.21 EXACT)
Expand Down
18 changes: 17 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,26 @@ release: #: Build the release version
$(MAKE) cmake BUILD_DIR=release BUILD_TYPE=Release && \
$(MAKE) build BUILD_DIR=release

min_debug: #: Minimal build with debugging symbols
minimal_debug: #: Minimal build with debugging symbols
$(MAKE) cmake BUILD_DIR=debug BUILD_TYPE=debug EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DVELOX_BUILD_MINIMAL=ON"
$(MAKE) build BUILD_DIR=debug

min_debug: minimal_debug

minimal: #: Minimal build
$(MAKE) cmake BUILD_DIR=release BUILD_TYPE=release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DVELOX_BUILD_MINIMAL=ON"
$(MAKE) build BUILD_DIR=release

dwio: #: Minimal build with dwio enabled.
$(MAKE) cmake BUILD_DIR=release BUILD_TYPE=release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} \
-DVELOX_BUILD_MINIMAL_WITH_DWIO=ON"
$(MAKE) build BUILD_DIR=release

dwio_debug: #: Minimal build with dwio debugging symbols.
$(MAKE) cmake BUILD_DIR=debug BUILD_TYPE=debug EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} \
-DVELOX_BUILD_MINIMAL_WITH_DWIO=ON"
$(MAKE) build BUILD_DIR=debug

benchmarks-basic-build:
$(MAKE) release EXTRA_CMAKE_FLAGS=" ${EXTRA_CMAKE_FLAGS} \
-DVELOX_BUILD_TESTING=OFF \
Expand Down
Loading

0 comments on commit 6e828fa

Please sign in to comment.