get_num_compressed() for SAPipelineMixin

Author: kshpv

tests/post_training/experimental/sparsify_activations/model_scope.py

@@ -9,14 +9,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
-from typing import Dict, List
-
-import nncf
 from nncf.experimental.torch.sparsify_activations import TargetScope
 from nncf.parameters import CompressWeightsMode
 from tests.post_training.experimental.sparsify_activations.pipelines import ImageClassificationTimmSparsifyActivations
 from tests.post_training.experimental.sparsify_activations.pipelines import LMSparsifyActivations
+from tests.post_training.model_scope import generate_tests_scope
 from tests.post_training.pipelines.base import BackendType
 
 SPARSIFY_ACTIVATIONS_MODELS = [
@@ -30,6 +27,7 @@
     {
         "reported_name": "tinyllama_ffn_sparse20",
         "model_id": "tinyllama/tinyllama-1.1b-step-50k-105b",
+        "model_name": "tinyllama",
         "pipeline_cls": LMSparsifyActivations,
         "compression_params": {
             "compress_weights": None,
@@ -45,6 +43,7 @@
     {
         "reported_name": "tinyllama_int8_asym_data_free_ffn_sparse20",
         "model_id": "tinyllama/tinyllama-1.1b-step-50k-105b",
+        "model_name": "tinyllama",
         "pipeline_cls": LMSparsifyActivations,
         "compression_params": {
             "compress_weights": {
@@ -62,6 +61,7 @@
     {
         "reported_name": "timm/deit3_small_patch16_224",
         "model_id": "deit3_small_patch16_224",
+        "model_name": "timm/deit3_small_patch16_224",
         "pipeline_cls": ImageClassificationTimmSparsifyActivations,
         "compression_params": {},
         "backends": [BackendType.FP32],
@@ -70,6 +70,7 @@
     {
         "reported_name": "timm/deit3_small_patch16_224_qkv_sparse20_fc1_sparse20_fc2_sparse30",
         "model_id": "deit3_small_patch16_224",
+        "model_name": "timm/deit3_small_patch16_224",
         "pipeline_cls": ImageClassificationTimmSparsifyActivations,
         "compression_params": {
             "sparsify_activations": {
@@ -85,34 +86,4 @@
 ]
 
 
-def generate_tests_scope(models_list: List[Dict]) -> Dict[str, Dict]:
-    """
-    Generate tests by names "{reported_name}_backend_{backend}"
-    """
-    tests_scope = {}
-    fp32_models = set()
-    for test_model_param in models_list:
-        model_id = test_model_param["model_id"]
-        reported_name = test_model_param["reported_name"]
-
-        for backend in test_model_param["backends"]:
-            model_param = copy.deepcopy(test_model_param)
-            if "is_batch_size_supported" not in model_param:  # Set default value of is_batch_size_supported.
-                model_param["is_batch_size_supported"] = True
-            test_case_name = f"{reported_name}_backend_{backend.value}"
-            model_param["backend"] = backend
-            model_param.pop("backends")
-            if backend == BackendType.FP32:
-                if model_id in fp32_models:
-                    msg = f"Duplicate test case for {model_id} with FP32 backend"
-                    raise nncf.ValidationError(msg)
-                fp32_models.add(model_id)
-            if test_case_name in tests_scope:
-                msg = f"{test_case_name} already in tests_scope"
-                raise nncf.ValidationError(msg)
-            tests_scope[test_case_name] = model_param
-
-    return tests_scope
-
-
 SPARSIFY_ACTIVATIONS_TEST_CASES = generate_tests_scope(SPARSIFY_ACTIVATIONS_MODELS)

tests/post_training/experimental/sparsify_activations/pipelines.py

@@ -11,7 +11,6 @@
 
 
 from dataclasses import dataclass
-from pathlib import Path
 from typing import Dict, List, Optional
 
 import numpy as np
@@ -33,10 +32,11 @@
 from nncf.torch.quantization.layers import INT8SymmetricWeightsDecompressor
 from tests.post_training.pipelines.base import PT_BACKENDS
 from tests.post_training.pipelines.base import BackendType
-from tests.post_training.pipelines.base import RunInfo
+from tests.post_training.pipelines.base import PTQTestPipeline
 from tests.post_training.pipelines.image_classification_timm import ImageClassificationTimm
 from tests.post_training.pipelines.lm_weight_compression import LMWeightCompression
 from tests.post_training.pipelines.lm_weight_compression import WCTimeStats
+from tests.torch.experimental.sparsify_activations.helpers import count_sparsifier_patterns_in_ov
 from tests.torch.helpers import set_torch_seed
 
 
@@ -52,40 +52,11 @@ class SATimeStats(WCTimeStats):
     REGEX_PREFIX = [*WCTimeStats.REGEX_PREFIX, SparsifyActivationsAlgoBackend.CALIBRATION_TRACKING_DESC]
 
 
-class SAPipelineMixin:
+class SAPipelineMixin(PTQTestPipeline):
     """
     Common methods in the test pipeline for Sparsify Activations.
     """
 
-    def __init__(
-        self,
-        reported_name: str,
-        model_id: str,
-        backend: BackendType,
-        compression_params: dict,
-        output_dir: Path,
-        data_dir: Path,
-        reference_data: dict,
-        no_eval: bool,
-        run_benchmark_app: bool,
-        params: dict = None,
-        batch_size: int = 1,
-    ):
-        super().__init__(
-            reported_name=reported_name,
-            model_id=model_id,
-            backend=backend,
-            compression_params=compression_params,
-            output_dir=output_dir,
-            data_dir=data_dir,
-            reference_data=reference_data,
-            no_eval=no_eval,
-            run_benchmark_app=run_benchmark_app,
-            params=params,
-            batch_size=batch_size,
-        )
-        self.run_info = RunInfo(model=reported_name, backend=backend)
-
     def collect_data_from_stdout(self, stdout: str):
         stats = SATimeStats()
         stats.fill(stdout)
@@ -111,6 +82,16 @@ def _compress(self):
                 **self.compression_params["sparsify_activations"],
             )
 
+    def get_num_compressed(self) -> None:
+        """
+
+        Get number of the FakeQuantize nodes in the compressed IR.
+        """
+        super().get_num_compressed()
+        ie = ov.Core()
+        model = ie.read_model(model=self.path_compressed_ir)
+        self.run_info.num_compress_nodes.num_sparse_activations = count_sparsifier_patterns_in_ov(model)
+
 
 class LMSparsifyActivations(SAPipelineMixin, LMWeightCompression):
     DEFAULT_SUBSET_SIZE = 32
@@ -198,6 +179,7 @@ def prepare_calibration_dataset(self):
         self.calibration_dataset = nncf.Dataset(chunks, self.get_transform_calibration_fn())
 
     def save_compressed_model(self):
+        self.path_compressed_ir = self.output_model_dir / self.OV_MODEL_NAME
         if self.backend == BackendType.CUDA_TORCH:
             self.model_hf.float()
             for module in self.model_hf.nncf.modules():

tests/post_training/experimental/sparsify_activations/test_sparsify_activations_conformance.py

@@ -18,8 +18,7 @@
 import yaml
 
 from tests.post_training.experimental.sparsify_activations.model_scope import SPARSIFY_ACTIVATIONS_TEST_CASES
-from tests.post_training.experimental.sparsify_activations.pipelines import SARunInfo
-from tests.post_training.pipelines.base import BackendType
+from tests.post_training.pipelines.base import RunInfo
 from tests.post_training.test_quantize_conformance import fixture_batch_size  # noqa: F401
 from tests.post_training.test_quantize_conformance import fixture_data  # noqa: F401
 from tests.post_training.test_quantize_conformance import fixture_extra_columns  # noqa: F401
@@ -44,7 +43,7 @@ def fixture_sparsify_activations_reference_data():
 
 @pytest.fixture(scope="session", name="sparsify_activations_result_data")
 def fixture_sparsify_activations_report_data(output_dir):
-    data: Dict[str, SARunInfo] = {}
+    data: Dict[str, RunInfo] = {}
     yield data
     if data:
         test_results = OrderedDict(sorted(data.items()))
@@ -59,7 +58,7 @@ def test_sparsify_activations(
     test_case_name: str,
     data_dir: Path,
     output_dir: Path,
-    sparsify_activations_result_data: Dict[str, SARunInfo],
+    sparsify_activations_result_data: Dict[str, RunInfo],
     no_eval: bool,
     batch_size: int,
     run_fp32_backend: bool,
@@ -69,9 +68,6 @@ def test_sparsify_activations(
     capsys: pytest.CaptureFixture,
     extra_columns: bool,
 ):
-    fp32_model_params = {
-        tc["model_id"]: tc for tc in SPARSIFY_ACTIVATIONS_TEST_CASES.values() if tc["backend"] == BackendType.FP32
-    }
     run_pipeline(
         test_case_name,
         sparsify_activations_reference_data,
@@ -90,5 +86,4 @@ def test_sparsify_activations(
         extra_columns,
         False,  # memory_monitor is not used in SA
         None,  # use_avx2 is not used in SA
-        fp32_model_params,
     )

tests/post_training/pipelines/base.py

@@ -30,7 +30,6 @@
 import nncf
 from nncf import TargetDevice
 from tests.cross_fw.shared.command import Command
-from tests.torch.experimental.sparsify_activations.helpers import count_sparsifier_patterns_in_ov
 from tools.memory_monitor import MemoryType
 from tools.memory_monitor import MemoryUnit
 from tools.memory_monitor import memory_monitor_context
@@ -207,7 +206,6 @@ def get_result_dict(self):
             "Status": self.status[:LIMIT_LENGTH_OF_STATUS] if self.status is not None else None,
             "Build url": os.environ.get("BUILD_URL", ""),
         }
-
         return result
 
 
@@ -423,9 +421,9 @@ def save_compressed_model(self) -> None:
             apply_moc_transformations(self.compressed_model, cf=True)
             ov.serialize(self.compressed_model, str(self.path_compressed_ir))
 
-    def get_num_compressed(self, to_count_sparse_activations: bool = False) -> None:
+    def get_num_compressed(self) -> None:
         """
-        to_count_sparse_activations
+
         Get number of the FakeQuantize nodes in the compressed IR.
         """
 
@@ -449,9 +447,6 @@ def get_num_compressed(self, to_count_sparse_activations: bool = False) -> None:
         self.run_info.num_compress_nodes.num_int8 = num_int8
         self.run_info.num_compress_nodes.num_int4 = num_int4
         self.run_info.num_compress_nodes.num_fq_nodes = num_fq
-        self.run_info.num_compress_nodes.num_sparse_activations = (
-            0 if not to_count_sparse_activations else count_sparsifier_patterns_in_ov(model)
-        )
 
     def run_bench(self) -> None:
         """

tests/post_training/test_quantize_conformance.py

@@ -178,6 +178,8 @@ def fixture_ptq_report_data(output_dir, run_benchmark_app, pytestconfig):
         if not run_benchmark_app:
             df = df.drop(columns=["FPS"])
 
+        df = df.drop(columns=["Num sparse activations"])
+
         output_dir.mkdir(parents=True, exist_ok=True)
         output_file = output_dir / "results.csv"
 
@@ -202,6 +204,7 @@ def fixture_wc_report_data(output_dir, run_benchmark_app, pytestconfig):
             df = df.drop(columns=["FPS"])
 
         df = df.drop(columns=["Num FQ"])
+        df = df.drop(columns=["Num sparse activations"])
 
         output_dir.mkdir(parents=True, exist_ok=True)
         output_file = output_dir / "results.csv"
@@ -266,7 +269,7 @@ def create_pipeline_kwargs(test_model_param, subset_size, test_case_name, refere
     print(f"PTQ params: {test_model_param['compression_params']}")
 
     # Get target fp32 metric value
-    model_name = test_case_name.split("_backend_")[0]
+    model_name = test_model_param.get("model_name", test_case_name.split("_backend_")[0])
     test_reference = reference_data[test_case_name]
     test_reference["metric_value_fp32"] = reference_data[f"{model_name}_backend_FP32"]["metric_value"]
 
@@ -297,11 +300,14 @@ def _update_status(pipeline: BaseTestPipeline, errors: List[ErrorReason]) -> Lis
     return unexpected_errors
 
 
-def _collect_errors(err_msg: str, pipeline: BaseTestPipeline) -> List[ErrorReason]:
+def _collect_errors(
+    pipeline: BaseTestPipeline,
+    exception_report: Optional[ErrorReport] = None,
+) -> List[ErrorReport]:
     errors = []
 
-    if err_msg:
-        errors.append(ErrorReport(ErrorReason.EXCEPTION, err_msg))
+    if exception_report:
+        errors.append(exception_report)
         return errors
 
     run_info = pipeline.run_info
@@ -372,9 +378,7 @@ def run_pipeline(
     memory_monitor: bool,
     use_avx2: Optional[bool] = None,
 ):
-    pipeline = None
-    err_msg = None
-    test_model_param = None
+    pipeline, exception_report, test_model_param = None, None, None
     start_time = time.perf_counter()
     if test_case_name not in reference_data:
         msg = f"{test_case_name} does not exist in 'reference_data.yaml'"
@@ -409,6 +413,7 @@ def run_pipeline(
         err_msg = str(e)
         if not err_msg:
             err_msg = "Unknown exception"
+        exception_report = ErrorReport(ErrorReason.EXCEPTION, err_msg)
         traceback.print_exc()
     finally:
         if pipeline is not None:
@@ -424,7 +429,7 @@ def run_pipeline(
             run_info = create_short_run_info(test_model_param, err_msg, test_case_name)
         run_info.time_total = time.perf_counter() - start_time
 
-        errors = _collect_errors(err_msg, pipeline)
+        errors = _collect_errors(pipeline, exception_report)
         unexpected_errors = _update_status(pipeline, errors)
         result_data[test_case_name] = run_info
 
@@ -495,7 +500,7 @@ def test_weight_compression(
         WC_TEST_CASES,
         wc_result_data,
         output_dir,
-        None,
+        None,  # data_dir is not used in WC
         no_eval,
         batch_size,
         run_fp32_backend,