output_processor only

Author: SimonLangerQC

src/datajudge/constraints/base.py

@@ -5,6 +5,8 @@
 
 import sqlalchemy as sa
 
+from datajudge.utils import OutputProcessor
+
 from ..db_access import DataReference
 from ..formatter import Formatter
 
@@ -119,10 +121,7 @@ def __init__(
         ref2=None,
         ref_value: Any = None,
         name: str = None,
-        output_processor: Callable[
-            [Collection, Optional[Collection]], Collection
-        ] = None,
-        output_remainder_slicer=slice(5),
+        output_processors: List[OutputProcessor] = None,
     ):
         self._check_if_valid_between_or_within(ref2, ref_value)
         self.ref = ref
@@ -133,8 +132,7 @@ def __init__(
         self.target_selections: OptionalSelections = None
         self.factual_queries: Optional[List[str]] = None
         self.target_queries: Optional[List[str]] = None
-        self.output_processor = output_processor
-        self.output_remainder_slicer = output_remainder_slicer
+        self.output_processors = output_processors
 
     def _check_if_valid_between_or_within(
         self, ref2: Optional[DataReference], ref_value: Optional[Any]
@@ -252,14 +250,10 @@ def test(self, engine: sa.engine.Engine) -> TestResult:
             target_queries,
         )
 
-    def apply_output_formatting_no_counts(
-        self, values: Collection, apply_remainder_limit=False
-    ) -> Collection:
-        if self.output_processor is not None:
-            values, _ = self.output_processor(values)  # type: ignore[call-arg]
-        if apply_remainder_limit:
-            values = list(values)
-            values = values[self.output_remainder_slicer]
+    def apply_output_formatting_no_counts(self, values: Collection) -> Collection:
+        if self.output_processors is not None:
+            for output_processor in self.output_processors:
+                values, _ = output_processor(values)
         return values
 
 

src/datajudge/constraints/miscs.py

@@ -130,12 +130,12 @@ def test(self, engine: sa.engine.Engine) -> TestResult:
             return TestResult.success()
 
         assertion_text = (
-            f"{self.ref} has violations of functional dependence (in total {len(violations)} rows), e.g. ({self.output_remainder_slicer}):\n"
+            f"{self.ref} has violations of functional dependence (in total {len(violations)} rows):\n"
             + "\n".join(
                 [
                     f"{violation}"
                     for violation in self.apply_output_formatting_no_counts(
-                        [tuple(elem) for elem in violations], True
+                        [tuple(elem) for elem in violations]
                     )
                 ]
             )

src/datajudge/constraints/uniques.py

@@ -9,7 +9,7 @@
 
 from .. import db_access
 from ..db_access import DataReference
-from ..utils import filternull_element
+from ..utils import OutputProcessor, filternull_element
 from .base import Constraint, OptionalSelections, T, TestResult, ToleranceGetter
 
 
@@ -94,11 +94,6 @@ class Uniques(Constraint, abc.ABC):
     The suggested function is ``datajudge.utils.output_processor_sort`` from this file,
     - see its documentation for details.
 
-    By default, the number of subset or superset remainders (excess or missing values)
-    for `UniquesSubset` and `UniquesSuperset` is sliced by [:5] (i.e. the first 5) in the assertion message.
-    This can be configured using `output_remainder_slicer`.
-    This argument does not have an effect for `UniquesEquality`.
-
     One use is of this constraint is to test for consistency in columns with expected
     categorical values.
     """
@@ -107,10 +102,7 @@ def __init__(
         self,
         ref: DataReference,
         name: str = None,
-        output_processor: Callable[
-            [Collection, Optional[Collection]], Collection
-        ] = None,
-        output_remainder_slicer=slice(5),
+        output_processors: List[OutputProcessor] = None,
         *,
         ref2: DataReference = None,
         uniques: Collection = None,
@@ -127,8 +119,7 @@ def __init__(
             ref2=ref2,
             ref_value=ref_value,
             name=name,
-            output_processor=output_processor,
-            output_remainder_slicer=output_remainder_slicer,
+            output_processors=output_processors,
         )
 
         if filter_func is None:
@@ -232,17 +223,16 @@ def compare(
             output_elemes, output_counts = list(remainder.keys()), list(
                 remainder.values()
             )
-            if self.output_processor is not None:
-                output_elemes, output_counts = self.output_processor(
-                    output_elemes, output_counts
-                )
-            output_elemes = output_elemes[self.output_remainder_slicer]
-            output_counts = output_counts[self.output_remainder_slicer]
+            if self.output_processors is not None:
+                for output_processor in self.output_processors:
+                    output_elemes, output_counts = output_processor(
+                        output_elemes, output_counts
+                    )
 
             assertion_text = (
                 f"{self.ref} has a fraction of {relative_violations} > "
                 f"{self.max_relative_violations} {'DISTINCT ' if self.compare_distinct else ''}values ({n_violations} / {n_rows}) not being an element of "
-                f"'{self.apply_output_formatting_no_counts(set(target_values))}'. It has e.g. ({self.output_remainder_slicer}) excess elements "
+                f"'{self.apply_output_formatting_no_counts(set(target_values))}'. It has excess elements "
                 f"'{output_elemes}' "
                 f"with counts {output_counts}."
                 f"{self.condition_string}"
@@ -277,8 +267,8 @@ def compare(
             assertion_text = (
                 f"{self.ref} has a fraction of "
                 f"{relative_violations} > {self.max_relative_violations} ({n_violations} / {n_rows}) "
-                f"lacking unique values of '{self.apply_output_formatting_no_counts(set(target_values))}'. E.g. ({self.output_remainder_slicer}) it "
-                f"doesn't have the unique value(s) '{self.apply_output_formatting_no_counts(list(remainder), apply_remainder_limit=True)}'."
+                f"lacking unique values of '{self.apply_output_formatting_no_counts(set(target_values))}'. It "
+                f"doesn't have the unique value(s) '{self.apply_output_formatting_no_counts(list(remainder))}'."
                 f"{self.condition_string}"
             )
             return False, assertion_text

src/datajudge/requirements.py

@@ -14,6 +14,8 @@
 
 import sqlalchemy as sa
 
+from datajudge.utils import OutputProcessor
+
 from .constraints import column as column_constraints
 from .constraints import date as date_constraints
 from .constraints import groupby as groupby_constraints
@@ -262,9 +264,7 @@ def add_uniques_equality_constraint(
         filter_func: Callable[[List[T]], List[T]] = None,
         map_func: Callable[[T], T] = None,
         reduce_func: Callable[[Collection], Collection] = None,
-        output_processor: Callable[
-            [Collection, Optional[Collection]], Collection
-        ] = None,
+        output_processors: List[OutputProcessor] = None,
         condition: Condition = None,
         name: str = None,
     ):
@@ -297,7 +297,7 @@ def add_uniques_equality_constraint(
                 filter_func=filter_func,
                 map_func=map_func,
                 reduce_func=reduce_func,
-                output_processor=output_processor,
+                output_processors=output_processors,
                 name=name,
             )
         )
@@ -312,10 +312,7 @@ def add_uniques_superset_constraint(
         reduce_func: Callable[[Collection], Collection] = None,
         condition: Condition = None,
         name: str = None,
-        output_processor: Callable[
-            [Collection, Optional[Collection]], Collection
-        ] = None,
-        output_remainder_slicer=slice(5),
+        output_processors: List[OutputProcessor] = None,
     ):
         """Check if unique values of columns are contained in the reference data.
 
@@ -341,7 +338,7 @@ def add_uniques_superset_constraint(
         categorical values.
 
         See ``Uniques`` for further details on ``map_func``, ``reduce_func``,
-        ``output_processor``, and ``output_remainder_slicer``.
+        and ``output_processors``.
         """
 
         ref = DataReference(self.data_source, columns, condition)
@@ -353,8 +350,7 @@ def add_uniques_superset_constraint(
                 filter_func=filter_func,
                 map_func=map_func,
                 reduce_func=reduce_func,
-                output_processor=output_processor,
-                output_remainder_slicer=output_remainder_slicer,
+                output_processors=output_processors,
                 name=name,
             )
         )
@@ -370,10 +366,7 @@ def add_uniques_subset_constraint(
         reduce_func: Callable[[Collection], Collection] = None,
         condition: Condition = None,
         name: str = None,
-        output_processor: Callable[
-            [Collection, Optional[Collection]], Collection
-        ] = None,
-        output_remainder_slicer=slice(5),
+        output_processors: List[OutputProcessor] = None,
     ):
         """Check if the data's unique values are contained in a given set of values.
 
@@ -403,7 +396,7 @@ def add_uniques_subset_constraint(
         or if `max_relative_violations` is 0.
 
         See ``Uniques`` for further details on ``map_func``, ``reduce_func``,
-        ``output_processor``, and ``output_remainder_slicer``.
+        and ``output_processors``.
         """
 
         ref = DataReference(self.data_source, columns, condition)
@@ -416,8 +409,7 @@ def add_uniques_subset_constraint(
                 compare_distinct=compare_distinct,
                 map_func=map_func,
                 reduce_func=reduce_func,
-                output_processor=output_processor,
-                output_remainder_slicer=output_remainder_slicer,
+                output_processors=output_processors,
                 name=name,
             )
         )
@@ -876,10 +868,7 @@ def add_functional_dependency_constraint(
         value_columns: List[str],
         condition: Condition = None,
         name: str = None,
-        output_processor: Callable[
-            [Collection, Optional[Collection]], Collection
-        ] = None,
-        output_remainder_slicer=slice(5),
+        output_processors: List[OutputProcessor] = None,
     ):
         """
         Expresses a functional dependency, a constraint where the `value_columns` are uniquely determined by the `key_columns`.
@@ -889,7 +878,8 @@ def add_functional_dependency_constraint(
         and all other columns are included `value_columns`.
         This constraint allows for a more general definition of functional dependencies, where the `key_columns` are not necessarily a primary key.
 
-        Additional configuration options (for details see the analogous parameters in for ``Uniques``-constraints) on how the output is sorted and how many counterexamples are shown are available as ``output_processor`` and ``output_remainder_slicer``.
+        An additional configuration option (for details see the analogous parameter in for ``Uniques``-constraints)
+        on how the output is sorted and how many counterexamples are shown is available as ``output_processors``.
 
         For more information on functional dependencies, see https://en.wikipedia.org/wiki/Functional_dependency.
         """
@@ -899,8 +889,7 @@ def add_functional_dependency_constraint(
             miscs_constraints.FunctionalDependency(
                 ref,
                 key_columns=key_columns,
-                output_processor=output_processor,
-                output_remainder_slicer=output_remainder_slicer,
+                output_processors=output_processors,
                 name=name,
             )
         )
@@ -1476,9 +1465,7 @@ def add_uniques_equality_constraint(
         filter_func: Callable[[List[T]], List[T]] = None,
         map_func: Callable[[T], T] = None,
         reduce_func: Callable[[Collection], Collection] = None,
-        output_processor: Callable[
-            [Collection, Optional[Collection]], Collection
-        ] = None,
+        output_processors: List[OutputProcessor] = None,
         condition1: Condition = None,
         condition2: Condition = None,
         name: str = None,
@@ -1513,7 +1500,7 @@ def add_uniques_equality_constraint(
                 filter_func=filter_func,
                 map_func=map_func,
                 reduce_func=reduce_func,
-                output_processor=output_processor,
+                output_processors=output_processors,
                 name=name,
             )
         )
@@ -1529,10 +1516,7 @@ def add_uniques_superset_constraint(
         condition1: Condition = None,
         condition2: Condition = None,
         name: str = None,
-        output_processor: Callable[
-            [Collection, Optional[Collection]], Collection
-        ] = None,
-        output_remainder_slicer=slice(5),
+        output_processors: List[OutputProcessor] = None,
     ):
         """Check if unique values of columns are contained in the reference data.
 
@@ -1559,7 +1543,7 @@ def add_uniques_superset_constraint(
         categorical values.
 
         See ``Uniques`` for further details on ``map_func``, ``reduce_func``,
-        ``output_processor``, and ``output_remainder_slicer``.
+        and ``output_processors``.
         """
 
         ref = DataReference(self.data_source, columns1, condition1)
@@ -1572,8 +1556,7 @@ def add_uniques_superset_constraint(
                 filter_func=filter_func,
                 map_func=map_func,
                 reduce_func=reduce_func,
-                output_processor=output_processor,
-                output_remainder_slicer=output_remainder_slicer,
+                output_processors=output_processors,
                 name=name,
             )
         )
@@ -1590,10 +1573,7 @@ def add_uniques_subset_constraint(
         condition1: Condition = None,
         condition2: Condition = None,
         name: str = None,
-        output_processor: Callable[
-            [Collection, Optional[Collection]], Collection
-        ] = None,
-        output_remainder_slicer=slice(5),
+        output_processors: List[OutputProcessor] = None,
     ):
         """Check if the given columns's unique values in are contained in reference data.
 
@@ -1622,7 +1602,7 @@ def add_uniques_subset_constraint(
         or if `max_relative_violations` is 0.
 
         See ``Uniques`` for further details on ``map_func``, ``reduce_func``,
-        ``output_processor``, and ``output_remainder_slicer``.
+        and ``output_processors``.
         """
 
         ref = DataReference(self.data_source, columns1, condition1)
@@ -1636,8 +1616,7 @@ def add_uniques_subset_constraint(
                 filter_func=filter_func,
                 map_func=map_func,
                 reduce_func=reduce_func,
-                output_processor=output_processor,
-                output_remainder_slicer=output_remainder_slicer,
+                output_processors=output_processors,
                 name=name,
             )
         )