Skip to content

Commit 9f64e42

Browse files
assignUserfacebook-github-bot
authored andcommitted
Enable Monolithic build (facebookincubator#9948)
Summary: This PR adds an option that combines all targets into a single `velox` target and build a single unified `libvelox.a`. This is entirely optional and defaults to off. This will make it easier for downstream users to use Velox by just linking to `velox::velox` in their own cmake instead of having to look through our cmake which targets they need. (Once we have a config etc.). I have yet to test the install functionality (it's strictly a bonus, the monolithic build is the goal for now). Pull Request resolved: facebookincubator#9948 Reviewed By: kgpai Differential Revision: D59974408 Pulled By: kevinwilfong fbshipit-source-id: 67459767d247a1264e1c5465c248586ac047ef4f
1 parent db784f6 commit 9f64e42

File tree

94 files changed

+781
-356
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+781
-356
lines changed

.cmake-format.yaml

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Currently this config mostly mirrors the default with the addition of custom functions
16+
format:
17+
line_width: 80
18+
tab_size: 2
19+
use_tabchars: false
20+
max_pargs_hwrap: 4
21+
max_subgroups_hwrap: 2
22+
min_prefix_chars: 4
23+
max_prefix_chars: 6
24+
separate_ctrl_name_with_space: false
25+
separate_fn_name_with_space: false
26+
dangle_parens: false
27+
command_case: "canonical"
28+
keyword_case: "unchanged"
29+
always_wrap:
30+
- set_target_properties
31+
- target_sources
32+
- target_link_libraries
33+
34+
parse:
35+
# We define these for our custom
36+
# functions so they get formatted correctly
37+
additional_commands:
38+
velox_add_library:
39+
pargs:
40+
nargs: 1+
41+
flags:
42+
- OBJECT
43+
- STATIC
44+
- SHARED
45+
- INTERFACE
46+
kwargs: {}
47+
48+
velox_base_add_library:
49+
pargs:
50+
nargs: 1+
51+
flags:
52+
- OBJECT
53+
- STATIC
54+
- SHARED
55+
- INTERFACE
56+
kwargs: {}
57+
58+
velox_compile_definitions:
59+
pargs: 1
60+
kwargs:
61+
PRIVATE: '*'
62+
PUBLIC: '*'
63+
INTERFACE: '*'
64+
65+
velox_include_directories:
66+
pargs: '1+'
67+
flags:
68+
- SYSTEM
69+
- BEFORE
70+
- AFTER
71+
kwargs:
72+
PRIVATE: '*'
73+
PUBLIC: '*'
74+
INTERFACE: '*'
75+
76+
velox_link_libraries:
77+
pargs: '1+'
78+
kwargs:
79+
PRIVATE: '*'
80+
PUBLIC: '*'
81+
INTERFACE: '*'
82+
83+
markup:
84+
first_comment_is_literal: true

.github/workflows/linux-build.yml

+1
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ jobs:
108108
"-DVELOX_ENABLE_ABFS=ON"
109109
"-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON"
110110
"-DVELOX_ENABLE_GPU=ON"
111+
"-DVELOX_MONO_LIBRARY=ON"
111112
)
112113
make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}"
113114

.github/workflows/preliminary_checks.yml

+4-2
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,15 @@ jobs:
3838
}
3939
- { name: "Code Format",
4040
command: "format-fix",
41-
message: "Found format issues",
41+
message: "Found format issues"
4242
}
4343
steps:
4444
- uses: actions/checkout@v4
4545
with:
4646
fetch-depth: 0
47-
47+
- run: |
48+
apt -y install python3-pip
49+
pip3 install --break-system-packages pyyaml
4850
- name: Fix git permissions
4951
# Usually actions/checkout does this but as we run in a container
5052
# it doesn't work

CMake/VeloxUtils.cmake

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
include_guard(GLOBAL)
15+
16+
# TODO use file sets
17+
function(velox_install_library_headers)
18+
# Find any headers and install them relative to the source tree in include.
19+
file(GLOB _hdrs "*.h")
20+
if(NOT "${_hdrs}" STREQUAL "")
21+
cmake_path(
22+
RELATIVE_PATH
23+
CMAKE_CURRENT_SOURCE_DIR
24+
BASE_DIRECTORY
25+
"${CMAKE_SOURCE_DIR}"
26+
OUTPUT_VARIABLE
27+
_hdr_dir)
28+
install(FILES ${_hdrs} DESTINATION include/${_hdr_dir})
29+
endif()
30+
endfunction()
31+
32+
# Base add velox library call to add a library and install it.
33+
function(velox_base_add_library TARGET)
34+
add_library(${TARGET} ${ARGN})
35+
install(TARGETS ${TARGET} DESTINATION lib/velox)
36+
velox_install_library_headers()
37+
endfunction()
38+
39+
# This is extremely hackish but presents an easy path to installation.
40+
function(velox_add_library TARGET)
41+
set(options OBJECT STATIC SHARED INTERFACE)
42+
set(oneValueArgs)
43+
set(multiValueArgs)
44+
cmake_parse_arguments(
45+
VELOX
46+
"${options}"
47+
"${oneValueArgs}"
48+
"${multiValueArgs}"
49+
${ARGN})
50+
51+
# Remove library type specifiers from ARGN
52+
set(library_type)
53+
if(VELOX_OBJECT)
54+
set(library_type OBJECT)
55+
elseif(VELOX_STATIC)
56+
set(library_type STATIC)
57+
elseif(VELOX_SHARED)
58+
set(library_type SHARED)
59+
elseif(VELOX_INTERFACE)
60+
set(library_type INTERFACE)
61+
endif()
62+
63+
list(REMOVE_ITEM ARGN OBJECT)
64+
list(REMOVE_ITEM ARGN STATIC)
65+
list(REMOVE_ITEM ARGN SHARED)
66+
list(REMOVE_ITEM ARGN INTERFACE)
67+
# Propagate to the underlying add_library and then install the target.
68+
if(VELOX_MONO_LIBRARY)
69+
if(TARGET velox)
70+
# Target already exists, append sources to it.
71+
target_sources(velox PRIVATE ${ARGN})
72+
else()
73+
# Create the target if this is the first invocation.
74+
add_library(velox ${ARGN})
75+
set_target_properties(velox PROPERTIES LIBRARY_OUTPUT_DIRECTORY
76+
${CMAKE_BINARY_DIR}/lib)
77+
set_target_properties(velox PROPERTIES ARCHIVE_OUTPUT_DIRECTORY
78+
${CMAKE_BINARY_DIR}/lib)
79+
install(TARGETS velox DESTINATION lib/velox)
80+
endif()
81+
# create alias for compatability
82+
if(NOT TARGET ${TARGET})
83+
add_library(${TARGET} ALIAS velox)
84+
endif()
85+
else()
86+
# Create a library for each invocation.
87+
velox_base_add_library(${TARGET} ${library_type} ${ARGN})
88+
endif()
89+
velox_install_library_headers()
90+
endfunction()
91+
92+
function(velox_link_libraries TARGET)
93+
# TODO(assignUser): Handle scope keywords (they currently are empty calls ala
94+
# target_link_libraries(target PRIVATE))
95+
if(VELOX_MONO_LIBRARY)
96+
message(DEBUG "${TARGET}: ${ARGN}")
97+
foreach(_lib ${ARGN})
98+
if("${_lib}" MATCHES "^velox_*")
99+
message(DEBUG "\t\tDROP: ${_lib}")
100+
else()
101+
message(DEBUG "\t\tADDING: ${_lib}")
102+
target_link_libraries(velox ${_lib})
103+
endif()
104+
endforeach()
105+
else()
106+
target_link_libraries(${TARGET} ${ARGN})
107+
endif()
108+
endfunction()
109+
110+
function(velox_include_directories TARGET)
111+
if(VELOX_MONO_LIBRARY)
112+
target_include_directories(velox ${ARGN})
113+
else()
114+
target_include_directories(${TARGET} ${ARGN})
115+
endif()
116+
endfunction()
117+
118+
function(velox_compile_definitions TARGET)
119+
if(VELOX_MONO_LIBRARY)
120+
target_compile_definitions(velox ${ARGN})
121+
else()
122+
target_compile_definitions(${TARGET} ${ARGN})
123+
endif()
124+
endfunction()
125+
126+
function(velox_sources TARGET)
127+
if(VELOX_MONO_LIBRARY)
128+
target_sources(velox ${ARGN})
129+
else()
130+
target_sources(${TARGET} ${ARGN})
131+
endif()
132+
endfunction()

CMakeLists.txt

+5-3
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ list(PREPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/CMake"
5050

5151
# Include our ThirdPartyToolchain dependencies macros
5252
include(ResolveDependency)
53+
include(VeloxUtils)
5354

5455
set_with_default(VELOX_DEPENDENCY_SOURCE_DEFAULT VELOX_DEPENDENCY_SOURCE AUTO)
5556
message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
@@ -68,6 +69,7 @@ option(
6869
"Build a minimal set of components, including DWIO (file format readers/writers).
6970
This will override other build options."
7071
OFF)
72+
option(VELOX_MONO_LIBRARY "Build single unified library." OFF)
7173

7274
# option() always creates a BOOL variable so we have to use a normal cache
7375
# variable with STRING type for this option.
@@ -94,9 +96,9 @@ option(VELOX_ENABLE_TPCH_CONNECTOR "Build TPC-H connector." ON)
9496
option(VELOX_ENABLE_PRESTO_FUNCTIONS "Build Presto SQL functions." ON)
9597
option(VELOX_ENABLE_SPARK_FUNCTIONS "Build Spark SQL functions." ON)
9698
option(VELOX_ENABLE_EXPRESSION "Build expression." ON)
97-
option(VELOX_ENABLE_EXAMPLES
98-
"Build examples. This will enable VELOX_ENABLE_EXPRESSION automatically."
99-
OFF)
99+
option(
100+
VELOX_ENABLE_EXAMPLES
101+
"Build examples. This will enable VELOX_ENABLE_EXPRESSION automatically." OFF)
100102
option(VELOX_ENABLE_SUBSTRAIT "Build Substrait-to-Velox converter." OFF)
101103
option(VELOX_ENABLE_BENCHMARKS "Enable Velox top level benchmarks." OFF)
102104
option(VELOX_ENABLE_BENCHMARKS_BASIC "Enable Velox basic benchmarks." OFF)

scripts/check.py

+9
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,15 @@ def fix(self, commit):
6363

6464

6565
class CMakeFormatter(str):
66+
def __init__(self, commit) -> None:
67+
super().__init__()
68+
try:
69+
import yaml
70+
except ModuleNotFoundError:
71+
# We need pyyaml so cmake-format can read '.cmake-format.yml'
72+
# otherwise it will run with default
73+
raise SystemExit("Please install 'pyyaml' for the CMake formatter.")
74+
6675
def diff(self, commit):
6776
return get_diff(
6877
self, util.run(f"cmake-format --first-comment-is-literal True {self}")[1]

scripts/setup-check.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ set -x
1919
export DEBIAN_FRONTEND=noninteractive
2020
apt update
2121
apt install --no-install-recommends -y clang-format-18 python3-pip git make ssh
22-
pip3 install --break-system-packages cmake==3.28.3 cmake_format black regex
22+
pip3 install --break-system-packages cmake==3.28.3 cmake_format black pyyaml regex
2323
pip3 cache purge
2424
apt purge --auto-remove -y python3-pip
2525
update-alternatives --install /usr/bin/clang-format clang-format "$(command -v clang-format-18)" 18

velox/buffer/CMakeLists.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
add_library(velox_buffer StringViewBufferHolder.cpp)
15+
velox_add_library(velox_buffer StringViewBufferHolder.cpp)
1616

17-
target_link_libraries(velox_buffer velox_memory velox_common_base Folly::folly)
17+
velox_link_libraries(velox_buffer velox_memory velox_common_base Folly::folly)
1818

1919
if(${VELOX_BUILD_TESTING})
2020
add_subdirectory(tests)

velox/common/base/CMakeLists.txt

+16-10
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,18 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
add_library(velox_exception Exceptions.cpp VeloxException.cpp Exceptions.h)
16-
target_link_libraries(
17-
velox_exception PUBLIC velox_flag_definitions velox_process Folly::folly
18-
fmt::fmt gflags::gflags glog::glog)
15+
velox_add_library(velox_exception Exceptions.cpp VeloxException.cpp
16+
Exceptions.h)
17+
velox_link_libraries(
18+
velox_exception
19+
PUBLIC velox_flag_definitions
20+
velox_process
21+
Folly::folly
22+
fmt::fmt
23+
gflags::gflags
24+
glog::glog)
1925

20-
add_library(
26+
velox_add_library(
2127
velox_common_base
2228
BitUtil.cpp
2329
Counters.cpp
@@ -32,7 +38,7 @@ add_library(
3238
StatsReporter.cpp
3339
SuccinctPrinter.cpp)
3440

35-
target_link_libraries(
41+
velox_link_libraries(
3642
velox_common_base
3743
PUBLIC velox_exception Folly::folly fmt::fmt xsimd
3844
PRIVATE velox_common_compression velox_process velox_test_util glog::glog)
@@ -45,8 +51,8 @@ if(${VELOX_ENABLE_BENCHMARKS})
4551
add_subdirectory(benchmarks)
4652
endif()
4753

48-
add_library(velox_id_map BigintIdMap.cpp)
49-
target_link_libraries(
54+
velox_add_library(velox_id_map BigintIdMap.cpp)
55+
velox_link_libraries(
5056
velox_id_map
5157
velox_memory
5258
velox_flag_definitions
@@ -56,8 +62,8 @@ target_link_libraries(
5662
fmt::fmt
5763
gflags::gflags)
5864

59-
add_library(velox_status Status.cpp)
60-
target_link_libraries(
65+
velox_add_library(velox_status Status.cpp)
66+
velox_link_libraries(
6167
velox_status
6268
PUBLIC fmt::fmt Folly::folly
6369
PRIVATE glog::glog)

velox/common/caching/CMakeLists.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
add_library(
15+
velox_add_library(
1616
velox_caching
1717
AsyncDataCache.cpp
1818
CacheTTLController.cpp
@@ -22,7 +22,7 @@ add_library(
2222
SsdFile.cpp
2323
SsdFileTracker.cpp
2424
StringIdMap.cpp)
25-
target_link_libraries(
25+
velox_link_libraries(
2626
velox_caching
2727
PUBLIC velox_common_base
2828
velox_exception

velox/common/compression/CMakeLists.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ if(${VELOX_BUILD_TESTING})
1616
add_subdirectory(tests)
1717
endif()
1818

19-
add_library(velox_common_compression Compression.cpp LzoDecompressor.cpp)
20-
target_link_libraries(
19+
velox_add_library(velox_common_compression Compression.cpp LzoDecompressor.cpp)
20+
velox_link_libraries(
2121
velox_common_compression
2222
PUBLIC Folly::folly
2323
PRIVATE velox_exception)

0 commit comments

Comments
 (0)