@@ -1756,7 +1756,6 @@ object PushDownPredicates extends Rule[LogicalPlan] {
1756
1756
object PushPredicateThroughNonJoin extends Rule [LogicalPlan ] with PredicateHelper {
1757
1757
def apply (plan : LogicalPlan ): LogicalPlan = plan transform applyLocally
1758
1758
1759
- val maxExprReusedInPushPredicate = conf.getConf(SQLConf .MAX_EXPRESSION_REUSED_IN_PUSH_PREDICATE )
1760
1759
val applyLocally : PartialFunction [LogicalPlan , LogicalPlan ] = {
1761
1760
// SPARK-13473: We can't push the predicate down when the underlying projection output non-
1762
1761
// deterministic field(s). Non-deterministic expressions are essentially stateful. This
@@ -1885,17 +1884,19 @@ object PushPredicateThroughNonJoin extends Rule[LogicalPlan] with PredicateHelpe
1885
1884
*/
1886
1885
def canCollapseExpression (
1887
1886
consumer : Expression ,
1888
- producerMap : Map [Attribute , Expression ]): Boolean = maxExprReusedInPushPredicate match {
1889
- case Int .MaxValue => true
1890
- case _ =>
1891
- CollapseProject .collectReferences(consumer)
1892
- .groupBy(identity)
1893
- .transform((_, v) => v.size)
1894
- .forall {
1895
- case (reference, count) =>
1896
- val producer = producerMap.getOrElse(reference, reference)
1897
- count <= maxExprReusedInPushPredicate || CollapseProject .isCheap(trimAliases(producer))
1898
- }
1887
+ producerMap : Map [Attribute , Expression ]): Boolean =
1888
+ conf.getConf(SQLConf .MAX_EXPRESSION_REUSED_IN_PUSH_PREDICATE ) match {
1889
+ case Int .MaxValue => true
1890
+ case maxReused =>
1891
+ CollapseProject .collectReferences(consumer)
1892
+ .groupBy(identity)
1893
+ .transform((_, v) => v.size)
1894
+ .forall {
1895
+ case (reference, count) =>
1896
+ val producer = producerMap.getOrElse(reference, reference)
1897
+ count <= maxReused || CollapseProject .isCheap(trimAliases(producer))
1898
+ }
1899
+
1899
1900
}
1900
1901
1901
1902
def canPushThrough (p : UnaryNode ): Boolean = p match {
0 commit comments