Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Add macro to refactor array functions registration #12577

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

refactor: Add macro to refactor array functions registration #12577

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ac0503f
refactor: Refactor array functions registeration
jinchengchenghh Mar 7, 2025
379897d
refactor varchar
jinchengchenghh Mar 7, 2025
6069c58
fix register
jinchengchenghh Mar 10, 2025
1197a9e
Update registered array functions
jinchengchenghh Mar 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions velox/functions/lib/RegistrationHelpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,36 @@ void registerUnaryNumeric(const std::vector<std::string>& aliases) {
registerUnaryFloatingPoint<T>(aliases);
}

#define REGISTER_NUMERIC_FUNCTIONS(functionName, prefix) \
{ \
functionName<int8_t>(prefix); \
functionName<int16_t>(prefix); \
functionName<int32_t>(prefix); \
functionName<int64_t>(prefix); \
functionName<int128_t>(prefix); \
functionName<float>(prefix); \
functionName<double>(prefix); \
}

#define REGISTER_SCALAR_FUNCTIONS(functionName, prefix) \
{ \
REGISTER_NUMERIC_FUNCTIONS(functionName, prefix) \
functionName<Varchar>(prefix); \
functionName<Varbinary>(prefix); \
functionName<bool>(prefix); \
functionName<Timestamp>(prefix); \
functionName<Date>(prefix); \
}

#define REGISTER_SCALAR_FUNCTIONS_WITHOUT_VARCHAR(functionName, prefix) \
{ \
REGISTER_NUMERIC_FUNCTIONS(functionName, prefix) \
functionName<Varbinary>(prefix); \
functionName<bool>(prefix); \
functionName<Timestamp>(prefix); \
functionName<Date>(prefix); \
}

} // namespace

} // namespace facebook::velox::functions
103 changes: 14 additions & 89 deletions velox/functions/prestosql/registration/ArrayFunctionsRegistration.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

#include "velox/functions/Registerer.h"
#include "velox/functions/lib/ArrayShuffle.h"
#include "velox/functions/lib/RegistrationHelpers.h"
#include "velox/functions/lib/Repeat.h"
#include "velox/functions/lib/Slice.h"
#include "velox/functions/prestosql/ArrayConstructor.h"
Expand Down Expand Up @@ -175,26 +176,14 @@ void registerArrayFunctions(const std::string& prefix) {
widthBucketArraySignature(),
makeWidthBucketArray);

registerArrayMinMaxFunctions<int8_t>(prefix);
registerArrayMinMaxFunctions<int16_t>(prefix);
registerArrayMinMaxFunctions<int32_t>(prefix);
registerArrayMinMaxFunctions<int64_t>(prefix);
registerArrayMinMaxFunctions<int128_t>(prefix);
registerArrayMinMaxFunctions<float>(prefix);
registerArrayMinMaxFunctions<double>(prefix);
REGISTER_NUMERIC_FUNCTIONS(registerArrayMinMaxFunctions, prefix);
registerArrayMinMaxFunctions<bool>(prefix);
registerArrayMinMaxFunctions<Varchar>(prefix);
registerArrayMinMaxFunctions<Timestamp>(prefix);
registerArrayMinMaxFunctions<Date>(prefix);
registerArrayMinMaxFunctions<Orderable<T1>>(prefix);

registerArrayJoinFunctions<int8_t>(prefix);
registerArrayJoinFunctions<int16_t>(prefix);
registerArrayJoinFunctions<int32_t>(prefix);
registerArrayJoinFunctions<int64_t>(prefix);
registerArrayJoinFunctions<int128_t>(prefix);
registerArrayJoinFunctions<float>(prefix);
registerArrayJoinFunctions<double>(prefix);
REGISTER_NUMERIC_FUNCTIONS(registerArrayJoinFunctions, prefix);
registerArrayJoinFunctions<bool>(prefix);
registerArrayJoinFunctions<Varchar>(prefix);
registerArrayJoinFunctions<Timestamp>(prefix);
Expand All @@ -213,96 +202,42 @@ void registerArrayFunctions(const std::string& prefix) {
Array<Generic<T1>>,
Array<Array<Generic<T1>>>>({prefix + "flatten"});

registerArrayRemoveFunctions<int8_t>(prefix);
registerArrayRemoveFunctions<int16_t>(prefix);
registerArrayRemoveFunctions<int32_t>(prefix);
registerArrayRemoveFunctions<int64_t>(prefix);
registerArrayRemoveFunctions<int128_t>(prefix);
registerArrayRemoveFunctions<float>(prefix);
registerArrayRemoveFunctions<double>(prefix);
registerArrayRemoveFunctions<bool>(prefix);
registerArrayRemoveFunctions<Timestamp>(prefix);
registerArrayRemoveFunctions<Date>(prefix);
registerArrayRemoveFunctions<Varbinary>(prefix);
registerArrayRemoveFunctions<Generic<T1>>(prefix);
REGISTER_SCALAR_FUNCTIONS_WITHOUT_VARCHAR(
registerArrayRemoveFunctions, prefix);
registerFunction<
ArrayRemoveFunctionString,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you know the reason why we need to implement a separate function specifically for the VARCHAR type?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The StringWriter push_back API uses copy_from which is a deep copy, usually in the simple function case, it should be a reference copy.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So we use setNoCopy instead of push_back for VARCHAR

Array<Varchar>,
Array<Varchar>,
Varchar>({prefix + "array_remove"});
registerArrayRemoveFunctions<Generic<T1>>(prefix);

registerArrayTrimFunctions<int8_t>(prefix);
registerArrayTrimFunctions<int16_t>(prefix);
registerArrayTrimFunctions<int32_t>(prefix);
registerArrayTrimFunctions<int64_t>(prefix);
registerArrayTrimFunctions<int128_t>(prefix);
registerArrayTrimFunctions<float>(prefix);
registerArrayTrimFunctions<double>(prefix);
registerArrayTrimFunctions<bool>(prefix);
registerArrayTrimFunctions<Timestamp>(prefix);
registerArrayTrimFunctions<Date>(prefix);
registerArrayTrimFunctions<Varbinary>(prefix);
REGISTER_SCALAR_FUNCTIONS_WITHOUT_VARCHAR(registerArrayTrimFunctions, prefix);
registerArrayTrimFunctions<Generic<T1>>(prefix);
registerFunction<
ArrayTrimFunctionString,
Array<Varchar>,
Array<Varchar>,
int64_t>({prefix + "trim_array"});

registerArrayTopNFunction<int8_t>(prefix);
registerArrayTopNFunction<int16_t>(prefix);
registerArrayTopNFunction<int32_t>(prefix);
registerArrayTopNFunction<int64_t>(prefix);
registerArrayTopNFunction<int128_t>(prefix);
registerArrayTopNFunction<float>(prefix);
registerArrayTopNFunction<double>(prefix);
REGISTER_NUMERIC_FUNCTIONS(registerArrayTopNFunction, prefix);
registerArrayTopNFunction<Varchar>(prefix);
registerArrayTopNFunction<Timestamp>(prefix);
registerArrayTopNFunction<Date>(prefix);
registerArrayTopNFunction<Varbinary>(prefix);
registerArrayTopNFunction<Orderable<T1>>(prefix);

registerArrayRemoveNullFunctions<int8_t>(prefix);
registerArrayRemoveNullFunctions<int16_t>(prefix);
registerArrayRemoveNullFunctions<int32_t>(prefix);
registerArrayRemoveNullFunctions<int64_t>(prefix);
registerArrayRemoveNullFunctions<int128_t>(prefix);
registerArrayRemoveNullFunctions<float>(prefix);
registerArrayRemoveNullFunctions<double>(prefix);
registerArrayRemoveNullFunctions<bool>(prefix);
registerArrayRemoveNullFunctions<Timestamp>(prefix);
registerArrayRemoveNullFunctions<Date>(prefix);
registerArrayRemoveNullFunctions<Varbinary>(prefix);
registerArrayRemoveNullFunctions<Generic<T1>>(prefix);
REGISTER_SCALAR_FUNCTIONS_WITHOUT_VARCHAR(
registerArrayRemoveNullFunctions, prefix);
registerFunction<
ArrayRemoveNullFunctionString,
Array<Varchar>,
Array<Varchar>>({prefix + "remove_nulls"});
registerArrayRemoveNullFunctions<Generic<T1>>(prefix);

registerArrayUnionFunctions<int8_t>(prefix);
registerArrayUnionFunctions<int16_t>(prefix);
registerArrayUnionFunctions<int32_t>(prefix);
registerArrayUnionFunctions<int64_t>(prefix);
registerArrayUnionFunctions<int128_t>(prefix);
registerArrayUnionFunctions<float>(prefix);
registerArrayUnionFunctions<double>(prefix);
registerArrayUnionFunctions<bool>(prefix);
registerArrayUnionFunctions<Timestamp>(prefix);
registerArrayUnionFunctions<Date>(prefix);
registerArrayUnionFunctions<Varbinary>(prefix);
REGISTER_SCALAR_FUNCTIONS(registerArrayUnionFunctions, prefix);
registerArrayUnionFunctions<Generic<T1>>(prefix);

registerArrayCombinationsFunctions<int8_t>(prefix);
registerArrayCombinationsFunctions<int16_t>(prefix);
registerArrayCombinationsFunctions<int32_t>(prefix);
registerArrayCombinationsFunctions<int64_t>(prefix);
registerArrayCombinationsFunctions<int128_t>(prefix);
registerArrayCombinationsFunctions<float>(prefix);
registerArrayCombinationsFunctions<double>(prefix);
registerArrayCombinationsFunctions<bool>(prefix);
registerArrayCombinationsFunctions<Varchar>(prefix);
registerArrayCombinationsFunctions<Timestamp>(prefix);
registerArrayCombinationsFunctions<Date>(prefix);
REGISTER_SCALAR_FUNCTIONS(registerArrayCombinationsFunctions, prefix);
registerArrayCombinationsFunctions<Generic<T1>>(prefix);

registerArrayCumSumFunction<int8_t>(prefix);
Expand All @@ -321,17 +256,7 @@ void registerArrayFunctions(const std::string& prefix) {
registerArrayHasDuplicatesFunctions<Varchar>(prefix);
registerArrayHasDuplicatesFunctions<Json>(prefix);

registerArrayFrequencyFunctions<bool>(prefix);
registerArrayFrequencyFunctions<int8_t>(prefix);
registerArrayFrequencyFunctions<int16_t>(prefix);
registerArrayFrequencyFunctions<int32_t>(prefix);
registerArrayFrequencyFunctions<int64_t>(prefix);
registerArrayFrequencyFunctions<int128_t>(prefix);
registerArrayFrequencyFunctions<float>(prefix);
registerArrayFrequencyFunctions<double>(prefix);
registerArrayFrequencyFunctions<Timestamp>(prefix);
registerArrayFrequencyFunctions<Date>(prefix);
registerArrayFrequencyFunctions<Varchar>(prefix);
REGISTER_SCALAR_FUNCTIONS(registerArrayFrequencyFunctions, prefix);

registerArrayNormalizeFunctions<int8_t>(prefix);
registerArrayNormalizeFunctions<int16_t>(prefix);
Expand Down
70 changes: 14 additions & 56 deletions velox/functions/sparksql/registration/RegisterArray.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,78 +89,35 @@ inline void registerArrayJoinFunctions(const std::string& prefix) {
}

template <typename T>
inline void registerArrayMinMaxFunctions(const std::string& prefix) {
void registerArrayMinMaxFunctions(const std::string& prefix) {
registerFunction<ArrayMinFunction, T, Array<T>>({prefix + "array_min"});
registerFunction<ArrayMaxFunction, T, Array<T>>({prefix + "array_max"});
}

inline void registerArrayMinMaxFunctions(const std::string& prefix) {
registerArrayMinMaxFunctions<int8_t>(prefix);
registerArrayMinMaxFunctions<int16_t>(prefix);
registerArrayMinMaxFunctions<int32_t>(prefix);
registerArrayMinMaxFunctions<int64_t>(prefix);
registerArrayMinMaxFunctions<int128_t>(prefix);
registerArrayMinMaxFunctions<float>(prefix);
registerArrayMinMaxFunctions<double>(prefix);
registerArrayMinMaxFunctions<bool>(prefix);
registerArrayMinMaxFunctions<Varchar>(prefix);
registerArrayMinMaxFunctions<Timestamp>(prefix);
registerArrayMinMaxFunctions<Date>(prefix);
}

template <typename T>
inline void registerArrayRemoveFunctions(const std::string& prefix) {
void registerArrayRemoveFunctions(const std::string& prefix) {
registerFunction<ArrayRemoveFunction, Array<T>, Array<T>, T>(
{prefix + "array_remove"});
}

inline void registerArrayRemoveFunctions(const std::string& prefix) {
registerArrayRemoveFunctions<int8_t>(prefix);
registerArrayRemoveFunctions<int16_t>(prefix);
registerArrayRemoveFunctions<int32_t>(prefix);
registerArrayRemoveFunctions<int64_t>(prefix);
registerArrayRemoveFunctions<int128_t>(prefix);
registerArrayRemoveFunctions<float>(prefix);
registerArrayRemoveFunctions<double>(prefix);
registerArrayRemoveFunctions<bool>(prefix);
registerArrayRemoveFunctions<Timestamp>(prefix);
registerArrayRemoveFunctions<Date>(prefix);
registerArrayRemoveFunctions<Varbinary>(prefix);
registerArrayRemoveFunctions<Generic<T1>>(prefix);
registerFunction<
ArrayRemoveFunctionString,
Array<Varchar>,
Array<Varchar>,
Varchar>({prefix + "array_remove"});
}

template <typename T>
inline void registerArrayUnionFunction(const std::string& prefix) {
void registerArrayUnionFunction(const std::string& prefix) {
registerFunction<ArrayUnionFunction, Array<T>, Array<T>, Array<T>>(
{prefix + "array_union"});
}

inline void registerArrayUnionFunctions(const std::string& prefix) {
registerArrayUnionFunction<int8_t>(prefix);
registerArrayUnionFunction<int16_t>(prefix);
registerArrayUnionFunction<int32_t>(prefix);
registerArrayUnionFunction<int64_t>(prefix);
registerArrayUnionFunction<int128_t>(prefix);
registerArrayUnionFunction<float>(prefix);
registerArrayUnionFunction<double>(prefix);
registerArrayUnionFunction<bool>(prefix);
registerArrayUnionFunction<Timestamp>(prefix);
registerArrayUnionFunction<Date>(prefix);
registerArrayUnionFunction<Varbinary>(prefix);
registerArrayUnionFunction<Varchar>(prefix);
registerArrayUnionFunction<Generic<T1>>(prefix);
}

void registerArrayFunctions(const std::string& prefix) {
registerArrayConcatFunctions(prefix);
registerArrayJoinFunctions(prefix);
registerArrayMinMaxFunctions(prefix);
registerArrayRemoveFunctions(prefix);
REGISTER_SCALAR_FUNCTIONS(registerArrayMinMaxFunctions, prefix);
REGISTER_SCALAR_FUNCTIONS_WITHOUT_VARCHAR(
registerArrayRemoveFunctions, prefix);
registerArrayRemoveFunctions<Generic<T1>>(prefix);
registerFunction<
ArrayRemoveFunctionString,
Array<Varchar>,
Array<Varchar>,
Varchar>({prefix + "array_remove"});
registerSparkArrayFunctions(prefix);
// Register array sort functions.
exec::registerStatefulVectorFunction(
Expand Down Expand Up @@ -195,7 +152,8 @@ void registerArrayFunctions(const std::string& prefix) {
Array<Generic<T1>>,
Array<Generic<T1>>,
Generic<T1>>({prefix + "array_append"});
registerArrayUnionFunctions(prefix);
REGISTER_SCALAR_FUNCTIONS(registerArrayUnionFunction, prefix);
registerArrayUnionFunction<Generic<T1>>(prefix);
}

} // namespace sparksql
Expand Down
Loading