[TESTS] Make a common entry point for conformance tests

tests/post_training/README.md

@@ -160,3 +160,14 @@ To mark a test as expected to fail (xfail) when a number of compression operatio
   ...
   num_compressed_xfail_reason: "Issue-<jira ticket number>"
 ```
+
+To mark a test as expected to fail (xfail) during the compression process with an exception:
+
+```yml
+<Name from model scopes>_backend_<BACKEND>:
+  ...
+    exception_xfail_reason:
+      type: "<ExceptionType>", e.g. TypeError
+      error_message: "<Error message from Exception>"
+      message: "Issue-<jira ticket number>"
+```

tests/post_training/conftest.py

@@ -9,6 +9,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from pathlib import Path
+
+import pytest
+
 
 def pytest_addoption(parser):
     parser.addoption("--data", action="store", help="Data directory")
@@ -35,3 +39,66 @@ def pytest_addoption(parser):
         help="Report memory using MemoryMonitor from tools/memory_monitor.py. "
         "Warning: currently, reported memory values are not always reproducible.",
     )
+
+
+@pytest.fixture(scope="session", name="data_dir")
+def fixture_data(pytestconfig):
+    if pytestconfig.getoption("data") is None:
+        msg = "This test requires the --data argument to be specified."
+        raise ValueError(msg)
+    return Path(pytestconfig.getoption("data"))
+
+
+@pytest.fixture(scope="session", name="output_dir")
+def fixture_output(pytestconfig):
+    return Path(pytestconfig.getoption("output"))
+
+
+@pytest.fixture(scope="session", name="no_eval")
+def fixture_no_eval(pytestconfig):
+    return pytestconfig.getoption("no_eval")
+
+
+@pytest.fixture(scope="session", name="batch_size")
+def fixture_batch_size(pytestconfig):
+    return pytestconfig.getoption("batch_size")
+
+
+@pytest.fixture(scope="session", name="subset_size")
+def fixture_subset_size(pytestconfig):
+    return pytestconfig.getoption("subset_size")
+
+
+@pytest.fixture(scope="session", name="run_fp32_backend")
+def fixture_run_fp32_backend(pytestconfig):
+    return pytestconfig.getoption("fp32")
+
+
+@pytest.fixture(scope="session", name="run_torch_cuda_backend")
+def fixture_run_torch_cuda_backend(pytestconfig):
+    return pytestconfig.getoption("cuda")
+
+
+@pytest.fixture(scope="session", name="run_benchmark_app")
+def fixture_run_benchmark_app(pytestconfig):
+    return pytestconfig.getoption("benchmark")
+
+
+@pytest.fixture(scope="session", name="torch_compile_validation")
+def fixture_torch_compile_validation(pytestconfig):
+    return pytestconfig.getoption("torch_compile_validation")
+
+
+@pytest.fixture(scope="session", name="extra_columns")
+def fixture_extra_columns(pytestconfig):
+    return pytestconfig.getoption("extra_columns")
+
+
+@pytest.fixture(scope="session", name="memory_monitor")
+def fixture_memory_monitor(pytestconfig):
+    return pytestconfig.getoption("memory_monitor")
+
+
+@pytest.fixture(scope="session", name="forked")
+def fixture_forked(pytestconfig):
+    return pytestconfig.getoption("forked")

tests/post_training/data/ptq_reference_data.yaml

@@ -18,7 +18,10 @@ hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_statefull_backend_OPTIMUM:
   metric_value: null
 hf/hf-internal-testing/tiny-random-GPTNeoXForCausalLM_stateless_backend_OPTIMUM:
   metric_value: null
-  xfail_reason: "Issue-161969"
+  exception_xfail_reason:
+    type: "TypeError"
+    error_message: "cannot pickle 'openvino._pyopenvino.Tensor' object"
+    message: "Issue-161969"
 hf/hf-internal-testing/tiny-random-gpt2_backend_FP32:
   metric_value: null
 hf/hf-internal-testing/tiny-random-gpt2_backend_OPTIMUM:

tests/post_training/experimental/sparsify_activations/model_scope.py

@@ -9,14 +9,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
-from typing import Dict, List
-
-import nncf
 from nncf.experimental.torch.sparsify_activations import TargetScope
 from nncf.parameters import CompressWeightsMode
 from tests.post_training.experimental.sparsify_activations.pipelines import ImageClassificationTimmSparsifyActivations
 from tests.post_training.experimental.sparsify_activations.pipelines import LMSparsifyActivations
+from tests.post_training.model_scope import generate_tests_scope
 from tests.post_training.pipelines.base import BackendType
 
 SPARSIFY_ACTIVATIONS_MODELS = [
@@ -30,6 +27,7 @@
     {
         "reported_name": "tinyllama_ffn_sparse20",
         "model_id": "tinyllama/tinyllama-1.1b-step-50k-105b",
+        "model_name": "tinyllama",
         "pipeline_cls": LMSparsifyActivations,
         "compression_params": {
             "compress_weights": None,
@@ -45,6 +43,7 @@
     {
         "reported_name": "tinyllama_int8_asym_data_free_ffn_sparse20",
         "model_id": "tinyllama/tinyllama-1.1b-step-50k-105b",
+        "model_name": "tinyllama",
         "pipeline_cls": LMSparsifyActivations,
         "compression_params": {
             "compress_weights": {
@@ -62,6 +61,7 @@
     {
         "reported_name": "timm/deit3_small_patch16_224",
         "model_id": "deit3_small_patch16_224",
+        "model_name": "timm/deit3_small_patch16_224",
         "pipeline_cls": ImageClassificationTimmSparsifyActivations,
         "compression_params": {},
         "backends": [BackendType.FP32],
@@ -70,6 +70,7 @@
     {
         "reported_name": "timm/deit3_small_patch16_224_qkv_sparse20_fc1_sparse20_fc2_sparse30",
         "model_id": "deit3_small_patch16_224",
+        "model_name": "timm/deit3_small_patch16_224",
         "pipeline_cls": ImageClassificationTimmSparsifyActivations,
         "compression_params": {
             "sparsify_activations": {
@@ -85,34 +86,4 @@
 ]
 
 
-def generate_tests_scope(models_list: List[Dict]) -> Dict[str, Dict]:
-    """
-    Generate tests by names "{reported_name}_backend_{backend}"
-    """
-    tests_scope = {}
-    fp32_models = set()
-    for test_model_param in models_list:
-        model_id = test_model_param["model_id"]
-        reported_name = test_model_param["reported_name"]
-
-        for backend in test_model_param["backends"]:
-            model_param = copy.deepcopy(test_model_param)
-            if "is_batch_size_supported" not in model_param:  # Set default value of is_batch_size_supported.
-                model_param["is_batch_size_supported"] = True
-            test_case_name = f"{reported_name}_backend_{backend.value}"
-            model_param["backend"] = backend
-            model_param.pop("backends")
-            if backend == BackendType.FP32:
-                if model_id in fp32_models:
-                    msg = f"Duplicate test case for {model_id} with FP32 backend"
-                    raise nncf.ValidationError(msg)
-                fp32_models.add(model_id)
-            if test_case_name in tests_scope:
-                msg = f"{test_case_name} already in tests_scope"
-                raise nncf.ValidationError(msg)
-            tests_scope[test_case_name] = model_param
-
-    return tests_scope
-
-
 SPARSIFY_ACTIVATIONS_TEST_CASES = generate_tests_scope(SPARSIFY_ACTIVATIONS_MODELS)