From 57076c94d8037dace9f1052739b98a9911778881 Mon Sep 17 00:00:00 2001 From: Zhichao Zhang Date: Mon, 10 Mar 2025 19:43:23 +0800 Subject: [PATCH] fix velox api --- .../backendsapi/velox/VeloxIteratorApi.scala | 28 +++++++++++++++---- .../Storages/SubstraitSource/CMakeLists.txt | 17 ++++++++--- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index 3f9bf77ae15c..e57eb7bbf177 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -65,7 +65,8 @@ class VeloxIteratorApi extends IteratorApi with Logging { fileSizes, modificationTimes, partitionColumns, - metadataColumns) = + metadataColumns, + otherMetadataColumns) = constructSplitInfo(partitionSchema, f.files, metadataColumnNames) val preferredLocations = SoftAffinity.getFilePartitionLocations(f) @@ -80,7 +81,8 @@ class VeloxIteratorApi extends IteratorApi with Logging { metadataColumns, fileFormat, preferredLocations.toList.asJava, - mapAsJavaMap(properties) + mapAsJavaMap(properties), + otherMetadataColumns ) case _ => throw new UnsupportedOperationException(s"Unsupported input partition.") @@ -104,7 +106,15 @@ class VeloxIteratorApi extends IteratorApi with Logging { }.toArray val locations = partitions.flatMap(p => SoftAffinity.getFilePartitionLocations(p.asInstanceOf[FilePartition])) - val (paths, starts, lengths, fileSizes, modificationTimes, partitionColumns, metadataColumns) = + val ( + paths, + starts, + lengths, + fileSizes, + modificationTimes, + partitionColumns, + metadataColumns, + otherMetadataColumns) = constructSplitInfo(partitionSchema, partitionFiles, metadataColumnNames) LocalFilesBuilder.makeLocalFiles( partitionIndex, @@ -117,7 +127,8 @@ class VeloxIteratorApi extends IteratorApi with Logging { metadataColumns, fileFormat, locations.toList.asJava, - mapAsJavaMap(properties) + mapAsJavaMap(properties), + otherMetadataColumns ) } @@ -194,7 +205,14 @@ class VeloxIteratorApi extends IteratorApi with Logging { otherMetadataColumns.add( SparkShimLoader.getSparkShims.getOtherConstantMetadataColumnValues(file)) } - (paths, starts, lengths, fileSizes, modificationTimes, partitionColumns, metadataColumns) + (paths, + starts, + lengths, + fileSizes, + modificationTimes, + partitionColumns, + metadataColumns, + otherMetadataColumns) } override def injectWriteFilesTempPath(path: String, fileName: String): Unit = { diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt b/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt index a41592ff7437..647fd0c11085 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt +++ b/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt @@ -27,13 +27,22 @@ target_compile_options( if(ENABLE_HDFS) target_link_libraries( substrait_source - PUBLIC boost::headers_only ch_contrib::protobuf clickhouse_common_io - ch_contrib::hdfs substrait ch_contrib::roaring ch_contrib::rapidjson) + PUBLIC boost::headers_only + ch_contrib::protobuf + clickhouse_common_io + ch_contrib::hdfs + substrait + ch_contrib::roaring + ch_contrib::rapidjson) else() target_link_libraries( substrait_source - PUBLIC boost::headers_only ch_contrib::protobuf clickhouse_common_io - substrait ch_contrib::roaring ch_contrib::rapidjson) + PUBLIC boost::headers_only + ch_contrib::protobuf + clickhouse_common_io + substrait + ch_contrib::roaring + ch_contrib::rapidjson) endif() target_include_directories( substrait_source SYSTEM BEFORE