Skip to content

Commit 56ef005

Browse files
committed
fix
1 parent 1498b09 commit 56ef005

File tree

1 file changed

+13
-12
lines changed
  • sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer

1 file changed

+13
-12
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala

+13-12
Original file line numberDiff line numberDiff line change
@@ -1756,7 +1756,6 @@ object PushDownPredicates extends Rule[LogicalPlan] {
17561756
object PushPredicateThroughNonJoin extends Rule[LogicalPlan] with PredicateHelper {
17571757
def apply(plan: LogicalPlan): LogicalPlan = plan transform applyLocally
17581758

1759-
val maxExprReusedInPushPredicate = conf.getConf(SQLConf.MAX_EXPRESSION_REUSED_IN_PUSH_PREDICATE)
17601759
val applyLocally: PartialFunction[LogicalPlan, LogicalPlan] = {
17611760
// SPARK-13473: We can't push the predicate down when the underlying projection output non-
17621761
// deterministic field(s). Non-deterministic expressions are essentially stateful. This
@@ -1885,17 +1884,19 @@ object PushPredicateThroughNonJoin extends Rule[LogicalPlan] with PredicateHelpe
18851884
*/
18861885
def canCollapseExpression(
18871886
consumer: Expression,
1888-
producerMap: Map[Attribute, Expression]): Boolean = maxExprReusedInPushPredicate match {
1889-
case Int.MaxValue => true
1890-
case _ =>
1891-
CollapseProject.collectReferences(consumer)
1892-
.groupBy(identity)
1893-
.transform((_, v) => v.size)
1894-
.forall {
1895-
case (reference, count) =>
1896-
val producer = producerMap.getOrElse(reference, reference)
1897-
count <= maxExprReusedInPushPredicate || CollapseProject.isCheap(trimAliases(producer))
1898-
}
1887+
producerMap: Map[Attribute, Expression]): Boolean =
1888+
conf.getConf(SQLConf.MAX_EXPRESSION_REUSED_IN_PUSH_PREDICATE) match {
1889+
case Int.MaxValue => true
1890+
case maxReused =>
1891+
CollapseProject.collectReferences(consumer)
1892+
.groupBy(identity)
1893+
.transform((_, v) => v.size)
1894+
.forall {
1895+
case (reference, count) =>
1896+
val producer = producerMap.getOrElse(reference, reference)
1897+
count <= maxReused || CollapseProject.isCheap(trimAliases(producer))
1898+
}
1899+
18991900
}
19001901

19011902
def canPushThrough(p: UnaryNode): Boolean = p match {

0 commit comments

Comments
 (0)