Skip to content

Commit 8bf9a7b

Browse files
zhztheplayerzhztheplayer
authored
[CORE] Rename acbo to ras (#5231)
1 parent 131f700 commit 8bf9a7b

File tree

77 files changed

+1312
-1314
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+1312
-1314
lines changed

‎.github/workflows/velox_docker.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -191,16 +191,16 @@ jobs:
191191
--local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
192192
&& GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
193193
--local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1
194-
- name: Run TPC-H / TPC-DS with ACBO
194+
- name: Run TPC-H / TPC-DS with RAS
195195
run: |
196196
echo "JAVA_HOME: $JAVA_HOME"
197197
cd $GITHUB_WORKSPACE/tools/gluten-it
198198
GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
199199
--local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
200-
--extra-conf=spark.gluten.sql.advanced.cbo.enabled=true \
200+
--extra-conf=spark.gluten.sql.ras.enabled=true \
201201
&& GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
202202
--local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
203-
--extra-conf=spark.gluten.sql.advanced.cbo.enabled=true
203+
--extra-conf=spark.gluten.sql.ras.enabled=true
204204
205205
# run-tpc-test-centos8-oom-randomkill:
206206
# needs: build-native-lib

‎backends-velox/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
</dependency>
5353
<dependency>
5454
<groupId>org.apache.gluten</groupId>
55-
<artifactId>gluten-cbo-common</artifactId>
55+
<artifactId>gluten-ras-common</artifactId>
5656
<version>${project.version}</version>
5757
<type>test-jar</type>
5858
<scope>test</scope>

‎backends-velox/src/test/scala/org/apache/gluten/planner/VeloxCboSuite.scala ‎backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala

+20-20
Original file line numberDiff line numberDiff line change
@@ -16,41 +16,41 @@
1616
*/
1717
package org.apache.gluten.planner
1818

19-
import org.apache.gluten.cbo.Cbo
20-
import org.apache.gluten.cbo.CboSuiteBase._
21-
import org.apache.gluten.cbo.path.CboPath
22-
import org.apache.gluten.cbo.property.PropertySet
23-
import org.apache.gluten.cbo.rule.{CboRule, Shape, Shapes}
2419
import org.apache.gluten.planner.property.GlutenProperties.Conventions
20+
import org.apache.gluten.ras.Ras
21+
import org.apache.gluten.ras.RasSuiteBase._
22+
import org.apache.gluten.ras.path.RasPath
23+
import org.apache.gluten.ras.property.PropertySet
24+
import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes}
2525

2626
import org.apache.spark.rdd.RDD
2727
import org.apache.spark.sql.catalyst.InternalRow
2828
import org.apache.spark.sql.catalyst.expressions.Attribute
2929
import org.apache.spark.sql.execution._
3030
import org.apache.spark.sql.test.SharedSparkSession
3131

32-
class VeloxCboSuite extends SharedSparkSession {
33-
import VeloxCboSuite._
32+
class VeloxRasSuite extends SharedSparkSession {
33+
import VeloxRasSuite._
3434

3535
test("C2R, R2C - basic") {
3636
val in = RowUnary(RowLeaf())
37-
val planner = newCbo().newPlanner(in)
37+
val planner = newRas().newPlanner(in)
3838
val out = planner.plan()
3939
assert(out == RowUnary(RowLeaf()))
4040
}
4141

4242
test("C2R, R2C - explicitly requires any properties") {
4343
val in = RowUnary(RowLeaf())
4444
val planner =
45-
newCbo().newPlanner(in, PropertySet(List(Conventions.ANY)))
45+
newRas().newPlanner(in, PropertySet(List(Conventions.ANY)))
4646
val out = planner.plan()
4747
assert(out == RowUnary(RowLeaf()))
4848
}
4949

5050
test("C2R, R2C - requires columnar output") {
5151
val in = RowUnary(RowLeaf())
5252
val planner =
53-
newCbo().newPlanner(in, PropertySet(List(Conventions.VANILLA_COLUMNAR)))
53+
newRas().newPlanner(in, PropertySet(List(Conventions.VANILLA_COLUMNAR)))
5454
val out = planner.plan()
5555
assert(out == RowToColumnarExec(RowUnary(RowLeaf())))
5656
}
@@ -59,18 +59,18 @@ class VeloxCboSuite extends SharedSparkSession {
5959
val in =
6060
ColumnarUnary(RowUnary(RowUnary(ColumnarUnary(RowUnary(RowUnary(ColumnarUnary(RowLeaf())))))))
6161
val planner =
62-
newCbo().newPlanner(in, PropertySet(List(Conventions.ROW_BASED)))
62+
newRas().newPlanner(in, PropertySet(List(Conventions.ROW_BASED)))
6363
val out = planner.plan()
6464
assert(out == ColumnarToRowExec(ColumnarUnary(
6565
RowToColumnarExec(RowUnary(RowUnary(ColumnarToRowExec(ColumnarUnary(RowToColumnarExec(
6666
RowUnary(RowUnary(ColumnarToRowExec(ColumnarUnary(RowToColumnarExec(RowLeaf()))))))))))))))
67-
val paths = planner.newState().memoState().collectAllPaths(CboPath.INF_DEPTH).toList
67+
val paths = planner.newState().memoState().collectAllPaths(RasPath.INF_DEPTH).toList
6868
val pathCount = paths.size
6969
assert(pathCount == 165)
7070
}
7171

7272
test("C2R, R2C - Row unary convertible to Columnar") {
73-
object ConvertRowUnaryToColumnar extends CboRule[SparkPlan] {
73+
object ConvertRowUnaryToColumnar extends RasRule[SparkPlan] {
7474
override def shift(node: SparkPlan): Iterable[SparkPlan] = node match {
7575
case RowUnary(child) => List(ColumnarUnary(child))
7676
case other => List.empty
@@ -82,24 +82,24 @@ class VeloxCboSuite extends SharedSparkSession {
8282
val in =
8383
ColumnarUnary(RowUnary(RowUnary(ColumnarUnary(RowUnary(RowUnary(ColumnarUnary(RowLeaf())))))))
8484
val planner =
85-
newCbo(List(ConvertRowUnaryToColumnar))
85+
newRas(List(ConvertRowUnaryToColumnar))
8686
.newPlanner(in, PropertySet(List(Conventions.ROW_BASED)))
8787
val out = planner.plan()
8888
assert(out == ColumnarToRowExec(ColumnarUnary(ColumnarUnary(ColumnarUnary(
8989
ColumnarUnary(ColumnarUnary(ColumnarUnary(ColumnarUnary(RowToColumnarExec(RowLeaf()))))))))))
90-
val paths = planner.newState().memoState().collectAllPaths(CboPath.INF_DEPTH).toList
90+
val paths = planner.newState().memoState().collectAllPaths(RasPath.INF_DEPTH).toList
9191
val pathCount = paths.size
9292
assert(pathCount == 1094)
9393
}
9494
}
9595

96-
object VeloxCboSuite {
97-
def newCbo(): Cbo[SparkPlan] = {
98-
GlutenOptimization().asInstanceOf[Cbo[SparkPlan]]
96+
object VeloxRasSuite {
97+
def newRas(): Ras[SparkPlan] = {
98+
GlutenOptimization().asInstanceOf[Ras[SparkPlan]]
9999
}
100100

101-
def newCbo(cboRules: Seq[CboRule[SparkPlan]]): Cbo[SparkPlan] = {
102-
GlutenOptimization(cboRules).asInstanceOf[Cbo[SparkPlan]]
101+
def newRas(RasRules: Seq[RasRule[SparkPlan]]): Ras[SparkPlan] = {
102+
GlutenOptimization(RasRules).asInstanceOf[Ras[SparkPlan]]
103103
}
104104

105105
case class RowLeaf() extends LeafExecNode {

‎docs/Configuration.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ You can add these configurations into spark-defaults.conf to enable or disable t
1919
| spark.memory.offHeap.size | To set up how much memory to be used for Java OffHeap.<br /> Please notice Gluten Plugin will leverage this setting to allocate memory space for native usage even offHeap is disabled. <br /> The value is based on your system and it is recommended to set it larger if you are facing Out of Memory issue in Gluten Plugin | 30G |
2020
| spark.sql.sources.useV1SourceList | Choose to use V1 source | avro |
2121
| spark.sql.join.preferSortMergeJoin | To turn off preferSortMergeJoin in Spark | false |
22-
| spark.plugins | To load Gluten's components by Spark's plug-in loader | org.apache.gluten.GlutenPlugin |
22+
| spark.plugins | To load Gluten's components by Spark's plug-in loader | org.apache.gluten.GlutenPlugin |
2323
| spark.shuffle.manager | To turn on Gluten Columnar Shuffle Plugin | org.apache.spark.shuffle.sort.ColumnarShuffleManager |
2424
| spark.gluten.enabled | Enable Gluten, default is true. Just an experimental property. Recommend to enable/disable Gluten through the setting for `spark.plugins`. | true |
25-
| spark.gluten.sql.advanced.cbo.enabled | Experimental: Enables Gluten's advanced CBO features during physical planning. E.g, More efficient fallback strategy, etc. The option can be turned on and off individually despite vanilla Spark's CBO settings. Note, Gluten's query optimizer may still adopt a subset of its advanced CBO capabilities even this option is off. Enabling it would cause Gluten consider using CBO for optimization more aggressively. Note, this feature is still in development and may not bring performance profits. | false |
25+
| spark.gluten.sql.ras.enabled | Experimental: Enables RAS (relation algebra selector) during physical planning to generate more efficient query plan. Note, this feature is still in development and may not bring performance profits. | false |
2626
| spark.gluten.sql.columnar.maxBatchSize | Number of rows to be processed in each batch. Default value is 4096. | 4096 |
2727
| spark.gluten.memory.isolation | (Experimental) Enable isolated memory mode. If true, Gluten controls the maximum off-heap memory can be used by each task to X, X = executor memory / max task slots. It's recommended to set true if Gluten serves concurrent queries within a single session, since not all memory Gluten allocated is guaranteed to be spillable. In the case, the feature should be enabled to avoid OOM. Note when true, setting spark.memory.storageFraction to a lower value is suggested since storage memory is considered non-usable by Gluten. | false |
2828
| spark.gluten.sql.columnar.scanOnly | When enabled, this config will overwrite all other operators' enabling, and only Scan and Filter pushdown will be offloaded to native. | false |

‎gluten-core/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
</dependency>
3737
<dependency>
3838
<groupId>org.apache.gluten</groupId>
39-
<artifactId>gluten-cbo-common</artifactId>
39+
<artifactId>gluten-ras-common</artifactId>
4040
<version>${project.version}</version>
4141
<scope>compile</scope>
4242
</dependency>

‎gluten-core/src/main/scala/org/apache/gluten/execution/LimitTransformer.scala

+3-3
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,12 @@ case class LimitTransformer(child: SparkPlan, offset: Long, count: Long)
6060
case c: TransformSupport => c.doTransform(context).root
6161
case _ => null
6262
}
63-
// If ACBO is enabled and is capable to move limit operators up and down among the plan nodes,
63+
// If RAS is enabled and is capable to move limit operators up and down among the plan nodes,
6464
// It might become possible that an independent limit gets to be transited to a order-by-limit.
6565
// In that case, we should tuning on the validation procedure. Either to move limit validation
66-
// To ACBO, or re-validate it in ACBO, or add properties or rules in ACBO to avoid such moves.
66+
// To RAS, or re-validate it in RAS, or add properties or rules in RAS to avoid such moves.
6767
//
68-
// It's not a issue for now since ACBO doesn't do such moves.
68+
// It's not a issue for now since RAS doesn't do such moves.
6969
val relNode = getRelNode(context, operatorId, offset, count, child.output, input, true)
7070

7171
doNativeValidation(context, relNode)

‎gluten-core/src/main/scala/org/apache/gluten/extension/ColumnarOverrides.scala

+3-3
Original file line numberDiff line numberDiff line change
@@ -225,8 +225,8 @@ case class ColumnarOverrideRules(session: SparkSession)
225225
*/
226226
private def transformRules(outputsColumnar: Boolean): List[SparkSession => Rule[SparkPlan]] = {
227227

228-
def maybeCbo(outputsColumnar: Boolean): List[SparkSession => Rule[SparkPlan]] = {
229-
if (GlutenConfig.getConf.enableAdvancedCbo) {
228+
def maybeRas(outputsColumnar: Boolean): List[SparkSession => Rule[SparkPlan]] = {
229+
if (GlutenConfig.getConf.enableRas) {
230230
return List(
231231
(_: SparkSession) => TransformPreOverrides(List(ImplementFilter()), List.empty),
232232
(session: SparkSession) => EnumeratedTransform(session, outputsColumnar),
@@ -251,7 +251,7 @@ case class ColumnarOverrideRules(session: SparkSession)
251251
(_: SparkSession) => AddTransformHintRule(),
252252
(_: SparkSession) => FallbackBloomFilterAggIfNeeded()
253253
) :::
254-
maybeCbo(outputsColumnar) :::
254+
maybeRas(outputsColumnar) :::
255255
List(
256256
(_: SparkSession) => RemoveNativeWriteFilesSortAndProject(),
257257
(spark: SparkSession) => RewriteTransformer(spark),

‎gluten-core/src/main/scala/org/apache/gluten/extension/columnar/EnumeratedTransform.scala

+8-8
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@
1616
*/
1717
package org.apache.gluten.extension.columnar
1818

19-
import org.apache.gluten.cbo.property.PropertySet
20-
import org.apache.gluten.cbo.rule.{CboRule, Shape, Shapes}
2119
import org.apache.gluten.extension.columnar.transform.{ImplementExchange, ImplementJoin, ImplementOthers, ImplementSingleNode}
2220
import org.apache.gluten.planner.GlutenOptimization
2321
import org.apache.gluten.planner.property.GlutenProperties
22+
import org.apache.gluten.ras.property.PropertySet
23+
import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes}
2424
import org.apache.gluten.utils.LogLevelUtil
2525

2626
import org.apache.spark.sql.SparkSession
@@ -32,13 +32,13 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean)
3232
with LogLevelUtil {
3333
import EnumeratedTransform._
3434

35-
private val cboRules = List(
36-
CboImplement(ImplementOthers()),
37-
CboImplement(ImplementExchange()),
38-
CboImplement(ImplementJoin())
35+
private val rasRules = List(
36+
RasImplement(ImplementOthers()),
37+
RasImplement(ImplementExchange()),
38+
RasImplement(ImplementJoin())
3939
)
4040

41-
private val optimization = GlutenOptimization(cboRules)
41+
private val optimization = GlutenOptimization(rasRules)
4242

4343
private val reqConvention = GlutenProperties.Conventions.ANY
4444
private val altConventions =
@@ -55,7 +55,7 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean)
5555
}
5656

5757
object EnumeratedTransform {
58-
private case class CboImplement(delegate: ImplementSingleNode) extends CboRule[SparkPlan] {
58+
private case class RasImplement(delegate: ImplementSingleNode) extends RasRule[SparkPlan] {
5959
override def shift(node: SparkPlan): Iterable[SparkPlan] = {
6060
val out = List(delegate.impl(node))
6161
out

‎gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transform/ImplementSingleNode.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,7 @@ object ImplementOthers {
295295
// Children will be preserved as they are as children of the output node.
296296
//
297297
// Do not look-up on children on the input node in this rule. Otherwise
298-
// it may break ACBO which would group all the possible input nodes to
298+
// it may break RAS which would group all the possible input nodes to
299299
// search for validate candidates.
300300
class ReplaceSingleNode() extends LogLevelUtil with Logging {
301301

‎gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala

+6-6
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,18 @@
1616
*/
1717
package org.apache.gluten.planner
1818

19-
import org.apache.gluten.cbo.{CboExplain, Optimization}
20-
import org.apache.gluten.cbo.rule.CboRule
2119
import org.apache.gluten.planner.cost.GlutenCostModel
2220
import org.apache.gluten.planner.metadata.GlutenMetadataModel
2321
import org.apache.gluten.planner.plan.GlutenPlanModel
2422
import org.apache.gluten.planner.property.GlutenPropertyModel
2523
import org.apache.gluten.planner.rule.GlutenRules
24+
import org.apache.gluten.ras.{Optimization, RasExplain}
25+
import org.apache.gluten.ras.rule.RasRule
2626

2727
import org.apache.spark.sql.execution.SparkPlan
2828

2929
object GlutenOptimization {
30-
private object GlutenExplain extends CboExplain[SparkPlan] {
30+
private object GlutenExplain extends RasExplain[SparkPlan] {
3131
override def describeNode(node: SparkPlan): String = node.nodeName
3232
}
3333

@@ -38,16 +38,16 @@ object GlutenOptimization {
3838
GlutenMetadataModel(),
3939
GlutenPropertyModel(),
4040
GlutenExplain,
41-
CboRule.Factory.reuse(GlutenRules()))
41+
RasRule.Factory.reuse(GlutenRules()))
4242
}
4343

44-
def apply(rules: Seq[CboRule[SparkPlan]]): Optimization[SparkPlan] = {
44+
def apply(rules: Seq[RasRule[SparkPlan]]): Optimization[SparkPlan] = {
4545
Optimization[SparkPlan](
4646
GlutenPlanModel(),
4747
GlutenCostModel(),
4848
GlutenMetadataModel(),
4949
GlutenPropertyModel(),
5050
GlutenExplain,
51-
CboRule.Factory.reuse(rules))
51+
RasRule.Factory.reuse(rules))
5252
}
5353
}

‎gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCost.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@
1616
*/
1717
package org.apache.gluten.planner.cost
1818

19-
import org.apache.gluten.cbo.Cost
19+
import org.apache.gluten.ras.Cost
2020

2121
case class GlutenCost(value: Long) extends Cost

0 commit comments

Comments
 (0)