From 51a7bd7bc45b9a6e2cd8ca51e489bf5b45d2643f Mon Sep 17 00:00:00 2001 From: beliefer Date: Thu, 13 Mar 2025 17:08:38 +0800 Subject: [PATCH] Investigate SQLConf --- .../gluten/backendsapi/clickhouse/CHConfig.scala | 5 ++--- .../scala/org/apache/gluten/config/VeloxConfig.scala | 7 ++----- .../extension/columnar/ColumnarRuleApplier.scala | 4 +--- .../gluten/extension/injector/GlutenInjector.scala | 4 ++-- .../scala/org/apache/gluten/config/GlutenConfig.scala | 11 +++++++---- 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConfig.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConfig.scala index 4720db3ad7a3..489286f3a50c 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConfig.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConfig.scala @@ -20,7 +20,6 @@ import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.network.util.ByteUnit -import org.apache.spark.sql.internal.SQLConf object CHConfig { private[clickhouse] val BACKEND_NAME: String = "ch" @@ -57,7 +56,7 @@ object CHConfig { def startWithSettingsPrefix(key: String): Boolean = key.startsWith(RUNTIME_SETTINGS) def removeSettingsPrefix(key: String): String = key.substring(RUNTIME_SETTINGS.length + 1) - def get: CHConfig = new CHConfig(SQLConf.get) + def get: CHConfig = new CHConfig() import GlutenConfig._ @@ -106,7 +105,7 @@ object CHConfig { .createWithDefault(false) } -class CHConfig(conf: SQLConf) extends GlutenConfig(conf) { +class CHConfig extends GlutenConfig { import CHConfig._ def enableOnePipelineMergeTreeWrite: Boolean = diff --git a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala index 9fd8064d6eff..4a22f6644506 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala @@ -19,12 +19,11 @@ package org.apache.gluten.config import org.apache.gluten.config.GlutenConfig.{buildConf, buildStaticConf, COLUMNAR_MAX_BATCH_SIZE} import org.apache.spark.network.util.ByteUnit -import org.apache.spark.sql.internal.SQLConf import java.util.Locale import java.util.concurrent.TimeUnit -class VeloxConfig(conf: SQLConf) extends GlutenConfig(conf) { +class VeloxConfig extends GlutenConfig { import VeloxConfig._ def veloxColumnarWindowType: String = getConf(COLUMNAR_VELOX_WINDOW_TYPE) @@ -64,9 +63,7 @@ class VeloxConfig(conf: SQLConf) extends GlutenConfig(conf) { object VeloxConfig { - def get: VeloxConfig = { - new VeloxConfig(SQLConf.get) - } + def get: VeloxConfig = new VeloxConfig() val COLUMNAR_VELOX_WINDOW_TYPE = buildConf("spark.gluten.sql.columnar.backend.velox.window.type") diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala index 725786550764..fc029650f151 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala @@ -31,8 +31,6 @@ object ColumnarRuleApplier { val session: SparkSession, val caller: CallerInfo, val outputsColumnar: Boolean) { - val glutenConf: GlutenConfig = { - new GlutenConfig(session.sessionState.conf) - } + val glutenConf: GlutenConfig = new GlutenConfig(session) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/injector/GlutenInjector.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/injector/GlutenInjector.scala index fa8704509e2a..80cdcc4eea73 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/injector/GlutenInjector.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/injector/GlutenInjector.scala @@ -44,8 +44,8 @@ class GlutenInjector private[injector] (control: InjectorControl) { } private def applier(session: SparkSession): ColumnarRuleApplier = { - val conf = new GlutenConfig(session.sessionState.conf) - if (conf.enableRas) { + val glutenConf = new GlutenConfig(session) + if (glutenConf.enableRas) { return ras.createApplier(session) } legacy.createApplier(session) diff --git a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala index a511a55607e0..656ab84abe29 100644 --- a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala @@ -18,6 +18,7 @@ package org.apache.gluten.config import org.apache.spark.internal.Logging import org.apache.spark.network.util.ByteUnit +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.internal.{GlutenConfigUtil, SQLConf, SQLConfProvider} import com.google.common.collect.ImmutableList @@ -33,9 +34,13 @@ case class GlutenNumaBindingInfo( totalCoreRange: Array[String] = null, numCoresPerExecutor: Int = -1) {} -class GlutenConfig(conf: SQLConf) extends Logging { +class GlutenConfig(sessionOpt: Option[SparkSession] = None) extends Logging { import GlutenConfig._ + def this(spark: SparkSession) = this(Some(spark)) + + val conf: SQLConf = sessionOpt.map(_.sessionState.conf).getOrElse(SQLConf.get) + private lazy val configProvider = new SQLConfProvider(conf) def getConf[T](entry: ConfigEntry[T]): T = { @@ -436,9 +441,7 @@ object GlutenConfig { val SPARK_SHUFFLE_SPILL_COMPRESS = "spark.shuffle.spill.compress" val SPARK_SHUFFLE_SPILL_COMPRESS_DEFAULT: Boolean = true - def get: GlutenConfig = { - new GlutenConfig(SQLConf.get) - } + def get: GlutenConfig = new GlutenConfig() def prefixOf(backendName: String): String = { GLUTEN_CONFIG_PREFIX + backendName