Skip to content

Commit e9887fb

Browse files
ArnavBalyanArnavBalyan
authored
[GLUTEN-8633][VL] Rewrite Spark tests for Gluten ColumnarRange (#8634)
1 parent aaa9fb1 commit e9887fb

File tree

8 files changed

+104
-0
lines changed

8 files changed

+104
-0
lines changed

‎gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala

+2
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
242242
.exclude("SPARK-24165: CaseWhen/If - nullability of nested types")
243243
.exclude("SPARK-27671: Fix analysis exception when casting null in nested field in struct")
244244
.exclude("summary")
245+
.excludeGlutenTest(
246+
"SPARK-27439: Explain result should match collected result after view change")
245247
.excludeGlutenTest("distributeBy and localSort")
246248
.excludeGlutenTest("describe")
247249
.excludeGlutenTest("Allow leading/trailing whitespace in string before casting")

‎gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala

+24
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import org.apache.spark.sql.internal.SQLConf
2828
import org.apache.spark.sql.test.SQLTestData.TestData2
2929
import org.apache.spark.sql.types.StringType
3030

31+
import java.io.ByteArrayOutputStream
3132
import java.nio.charset.StandardCharsets
3233

3334
import scala.util.Random
@@ -359,6 +360,29 @@ class GlutenDataFrameSuite extends DataFrameSuite with GlutenSQLTestsTrait {
359360
checkResult(df, expectedBinaryResult)
360361
}
361362

363+
testGluten("SPARK-27439: Explain result should match collected result after view change") {
364+
withTempView("test", "test2", "tmp") {
365+
spark.range(10).createOrReplaceTempView("test")
366+
spark.range(5).createOrReplaceTempView("test2")
367+
spark.sql("select * from test").createOrReplaceTempView("tmp")
368+
val df = spark.sql("select * from tmp")
369+
spark.sql("select * from test2").createOrReplaceTempView("tmp")
370+
371+
val captured = new ByteArrayOutputStream()
372+
Console.withOut(captured) {
373+
df.explain(extended = true)
374+
}
375+
checkAnswer(df, spark.range(10).toDF)
376+
val output = captured.toString
377+
assert(output.contains("""== Parsed Logical Plan ==
378+
|'Project [*]
379+
|+- 'UnresolvedRelation [tmp]""".stripMargin))
380+
assert(output.contains("""== Physical Plan ==
381+
|*(1) ColumnarToRow
382+
|+- ColumnarRange 0, 10, 1, 2, 10""".stripMargin))
383+
}
384+
}
385+
362386
private def withExpr(newExpr: Expression): Column = new Column(newExpr)
363387

364388
def equalizer(expr: Expression, other: Any): Column = withExpr {

‎gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala

+2
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
264264
.exclude("SPARK-24165: CaseWhen/If - nullability of nested types")
265265
.exclude("SPARK-27671: Fix analysis exception when casting null in nested field in struct")
266266
.exclude("summary")
267+
.excludeGlutenTest(
268+
"SPARK-27439: Explain result should match collected result after view change")
267269
.excludeGlutenTest("distributeBy and localSort")
268270
.excludeGlutenTest("describe")
269271
.excludeGlutenTest("Allow leading/trailing whitespace in string before casting")

‎gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala

+24
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import org.apache.spark.sql.internal.SQLConf
2828
import org.apache.spark.sql.test.SQLTestData.TestData2
2929
import org.apache.spark.sql.types.StringType
3030

31+
import java.io.ByteArrayOutputStream
3132
import java.nio.charset.StandardCharsets
3233

3334
import scala.util.Random
@@ -360,6 +361,29 @@ class GlutenDataFrameSuite extends DataFrameSuite with GlutenSQLTestsTrait {
360361
checkResult(df, expectedBinaryResult)
361362
}
362363

364+
testGluten("SPARK-27439: Explain result should match collected result after view change") {
365+
withTempView("test", "test2", "tmp") {
366+
spark.range(10).createOrReplaceTempView("test")
367+
spark.range(5).createOrReplaceTempView("test2")
368+
spark.sql("select * from test").createOrReplaceTempView("tmp")
369+
val df = spark.sql("select * from tmp")
370+
spark.sql("select * from test2").createOrReplaceTempView("tmp")
371+
372+
val captured = new ByteArrayOutputStream()
373+
Console.withOut(captured) {
374+
df.explain(extended = true)
375+
}
376+
checkAnswer(df, spark.range(10).toDF)
377+
val output = captured.toString
378+
assert(output.contains("""== Parsed Logical Plan ==
379+
|'Project [*]
380+
|+- 'UnresolvedRelation [tmp]""".stripMargin))
381+
assert(output.contains("""== Physical Plan ==
382+
|*(1) ColumnarToRow
383+
|+- ColumnarRange 0, 10, 1, 2, 10""".stripMargin))
384+
}
385+
}
386+
363387
private def withExpr(newExpr: Expression): Column = new Column(newExpr)
364388

365389
def equalizer(expr: Expression, other: Any): Column = withExpr {

‎gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala

+2
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
262262
.exclude("SPARK-24165: CaseWhen/If - nullability of nested types")
263263
.exclude("SPARK-27671: Fix analysis exception when casting null in nested field in struct")
264264
.exclude("summary")
265+
.excludeGlutenTest(
266+
"SPARK-27439: Explain result should match collected result after view change")
265267
.excludeGlutenTest("distributeBy and localSort")
266268
.excludeGlutenTest("describe")
267269
.excludeGlutenTest("Allow leading/trailing whitespace in string before casting")

‎gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala

+24
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import org.apache.spark.sql.internal.SQLConf
2828
import org.apache.spark.sql.test.SQLTestData.TestData2
2929
import org.apache.spark.sql.types.StringType
3030

31+
import java.io.ByteArrayOutputStream
3132
import java.nio.charset.StandardCharsets
3233

3334
import scala.util.Random
@@ -360,6 +361,29 @@ class GlutenDataFrameSuite extends DataFrameSuite with GlutenSQLTestsTrait {
360361
checkResult(df, expectedBinaryResult)
361362
}
362363

364+
testGluten("SPARK-27439: Explain result should match collected result after view change") {
365+
withTempView("test", "test2", "tmp") {
366+
spark.range(10).createOrReplaceTempView("test")
367+
spark.range(5).createOrReplaceTempView("test2")
368+
spark.sql("select * from test").createOrReplaceTempView("tmp")
369+
val df = spark.sql("select * from tmp")
370+
spark.sql("select * from test2").createOrReplaceTempView("tmp")
371+
372+
val captured = new ByteArrayOutputStream()
373+
Console.withOut(captured) {
374+
df.explain(extended = true)
375+
}
376+
checkAnswer(df, spark.range(10).toDF)
377+
val output = captured.toString
378+
assert(output.contains("""== Parsed Logical Plan ==
379+
|'Project [*]
380+
|+- 'UnresolvedRelation [tmp]""".stripMargin))
381+
assert(output.contains("""== Physical Plan ==
382+
|*(1) ColumnarToRow
383+
|+- ColumnarRange 0, 10, 1, 2, 10""".stripMargin))
384+
}
385+
}
386+
363387
private def withExpr(newExpr: Expression): Column = new Column(newExpr)
364388

365389
def equalizer(expr: Expression, other: Any): Column = withExpr {

‎gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala

+2
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
546546
.excludeCH("SPARK-28067: Aggregate sum should not return wrong results for decimal overflow")
547547
.excludeCH("SPARK-35955: Aggregate avg should not return wrong results for decimal overflow")
548548
.excludeCH("summary")
549+
.excludeGlutenTest(
550+
"SPARK-27439: Explain result should match collected result after view change")
549551
.excludeCH(
550552
"SPARK-8608: call `show` on local DataFrame with random columns should return same value")
551553
.excludeCH(

‎gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala

+24
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import org.apache.spark.sql.internal.SQLConf
2828
import org.apache.spark.sql.test.SQLTestData.TestData2
2929
import org.apache.spark.sql.types.StringType
3030

31+
import java.io.ByteArrayOutputStream
3132
import java.nio.charset.StandardCharsets
3233

3334
import scala.util.Random
@@ -371,6 +372,29 @@ class GlutenDataFrameSuite extends DataFrameSuite with GlutenSQLTestsTrait {
371372
}
372373
}
373374

375+
testGluten("SPARK-27439: Explain result should match collected result after view change") {
376+
withTempView("test", "test2", "tmp") {
377+
spark.range(10).createOrReplaceTempView("test")
378+
spark.range(5).createOrReplaceTempView("test2")
379+
spark.sql("select * from test").createOrReplaceTempView("tmp")
380+
val df = spark.sql("select * from tmp")
381+
spark.sql("select * from test2").createOrReplaceTempView("tmp")
382+
383+
val captured = new ByteArrayOutputStream()
384+
Console.withOut(captured) {
385+
df.explain(extended = true)
386+
}
387+
checkAnswer(df, spark.range(10).toDF)
388+
val output = captured.toString
389+
assert(output.contains("""== Parsed Logical Plan ==
390+
|'Project [*]
391+
|+- 'UnresolvedRelation [tmp]""".stripMargin))
392+
assert(output.contains("""== Physical Plan ==
393+
|*(1) ColumnarToRow
394+
|+- ColumnarRange 0, 10, 1, 2, 10""".stripMargin))
395+
}
396+
}
397+
374398
private def withExpr(newExpr: Expression): Column = new Column(newExpr)
375399

376400
def equalizer(expr: Expression, other: Any): Column = withExpr {

0 commit comments

Comments
 (0)