Skip to content

Commit 4bbcefb

Browse files
AlexanderDokuchaevAlexanderDokuchaev
authored
[GA] test examples (#3001)
### Changes Add GA workflow to run tests examples. Set environment variables to get host-independent in the results: ``` env["ONEDNN_MAX_CPU_ISA"] = "AVX2" env["CUDA_VISIBLE_DEVICES"] = "" ``` ### Related tickets 153650 ### Tests https://github.com/openvinotoolkit/nncf/actions/runs/11332052082
1 parent e81a1a3 commit 4bbcefb

File tree

8 files changed

+138
-26
lines changed

8 files changed

+138
-26
lines changed

‎.github/workflows/examples.yml

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: Test examples
2+
permissions: read-all
3+
4+
on:
5+
schedule:
6+
- cron: '0 0 * * *'
7+
workflow_dispatch:
8+
inputs:
9+
pull_request_number:
10+
description: 'The pull request number'
11+
required: true
12+
type: number
13+
pytest_args:
14+
description: 'Pytest arguments'
15+
default: ''
16+
type: string
17+
18+
jobs:
19+
examples-cpu:
20+
name: Test exmaples CPU [${{ matrix.group }}/4]
21+
runs-on: ubuntu-22.04-16-cores
22+
strategy:
23+
fail-fast: false
24+
matrix:
25+
group: [1, 2, 3, 4]
26+
defaults:
27+
run:
28+
shell: bash
29+
steps:
30+
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
31+
with:
32+
lfs: true
33+
fetch-depth: 0 # Fetch full history to allow checking out any branch or PR
34+
- name: Fetch and Checkout the Pull Request Branch
35+
if: ${{ github.event_name == 'workflow_dispatch' }}
36+
run: |
37+
git fetch origin pull/${{ github.event.inputs.pull_request_number }}/head:pr-${{ github.event.inputs.pull_request_number }}
38+
git checkout pr-${{ github.event.inputs.pull_request_number }}
39+
- uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
40+
with:
41+
python-version: 3.10.14
42+
cache: pip
43+
- name: cpuinfo
44+
run: cat /proc/cpuinfo
45+
- name: Install NNCF and test requirements
46+
run: |
47+
pip install -e .
48+
pip install -r tests/cross_fw/examples/requirements.txt
49+
- name: Print installed modules
50+
run: pip list
51+
- name: Run examples test scope
52+
continue-on-error: true
53+
run: |
54+
python -m pytest -ras tests/cross_fw/examples \
55+
--junit-xml=pytest-results-${{ matrix.group }}.xml \
56+
--durations-path=tests/cross_fw/examples/.test_durations \
57+
--splitting-algorithm=least_duration \
58+
--splits 4 \
59+
--group ${{ matrix.group }} \
60+
${{ github.event.inputs.pytest_args || '' }}
61+
env:
62+
TQDM_DISABLE: 1
63+
- name: Upload artifact
64+
uses: actions/upload-artifact@v4
65+
with:
66+
name: pytest-results-${{ matrix.group }}
67+
path: pytest-results-${{ matrix.group }}.xml
68+
overwrite: True

‎.github/workflows/precommit.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ on:
1010
paths-ignore:
1111
- '**/*.md'
1212
- 'docs/**/*'
13-
- 'tests/post_training/*' # post_training tests are only run in Jenkins
13+
- 'tests/post_training/*' # post_training tests runs on Jenkins
1414
- 'tests/torch/sota_checkpoints_eval.json' # reference for PT e2e
1515
- 'tests/tensorflow/sota_checkpoints_eval.json' # reference for TF e2e
16-
16+
- 'tests/cross_fw/examples/*' # examples tests runs in separate workflow
1717
jobs:
1818
common:
1919
runs-on: ubuntu-20.04

‎constraints.txt

+1
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,4 @@ pytest-dependency==0.6.0
2121
pytest-ordering==0.6
2222
pytest-xdist==3.5.0
2323
pytest-forked==1.6.0
24+
pytest-split==0.9.0

‎tests/cross_fw/examples/.test_durations

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"tests/cross_fw/examples/test_examples.py::test_examples[llm_compression]": 222.974,
3+
"tests/cross_fw/examples/test_examples.py::test_examples[llm_compression_synthetic]": 873.780,
4+
"tests/cross_fw/examples/test_examples.py::test_examples[llm_tune_params]": 1018.932,
5+
"tests/cross_fw/examples/test_examples.py::test_examples[post_training_quantization_onnx_mobilenet_v2]": 178.509,
6+
"tests/cross_fw/examples/test_examples.py::test_examples[post_training_quantization_onnx_yolo8_quantize_with_accuracy_control]": 292.766,
7+
"tests/cross_fw/examples/test_examples.py::test_examples[post_training_quantization_openvino_anomaly_stfpm_quantize_with_accuracy_control]": 443.025,
8+
"tests/cross_fw/examples/test_examples.py::test_examples[post_training_quantization_openvino_mobilenet_v2_quantize]": 169.789,
9+
"tests/cross_fw/examples/test_examples.py::test_examples[post_training_quantization_openvino_yolo8_quantize]": 170.593,
10+
"tests/cross_fw/examples/test_examples.py::test_examples[post_training_quantization_openvino_yolo8_quantize_with_accuracy_control]": 205.533,
11+
"tests/cross_fw/examples/test_examples.py::test_examples[post_training_quantization_tensorflow_mobilenet_v2]": 149.202,
12+
"tests/cross_fw/examples/test_examples.py::test_examples[post_training_quantization_torch_mobilenet_v2]": 192.227,
13+
"tests/cross_fw/examples/test_examples.py::test_examples[post_training_quantization_torch_ssd300_vgg16]": 231.613,
14+
"tests/cross_fw/examples/test_examples.py::test_examples[quantization_aware_training_torch_anomalib]": 478.797,
15+
"tests/cross_fw/examples/test_examples.py::test_examples[quantization_aware_training_torch_resnet18]": 1251.144
16+
}

‎tests/cross_fw/examples/README.md

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Test examples
2+
3+
## Manual trigger
4+
5+
To manual run job use GitHub workflow:
6+
7+
https://github.com/openvinotoolkit/nncf/actions/workflows/examples.yml
8+
9+
Parameters:
10+
11+
- `pull_request_number`: The pull request number.
12+
- `pytest_args`: Additional pytest arguments (example `-k llm`)
13+
14+
## Parallel test
15+
16+
To set up parallel testing between jobs using `pytest-split` with the option `--splitting-algorithm=least_duration`,
17+
you need to ensure that each test's duration is tracked correctly and stored so that future test runs can use that data
18+
to split the tests efficiently. After run workflow get time frm artifact of job and
19+
add new test to [.test_durations](.test_durations).

‎tests/cross_fw/examples/example_scope.json

+21-19
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,11 @@
3838
"backend": "openvino",
3939
"requirements": "examples/post_training_quantization/openvino/yolov8/requirements.txt",
4040
"cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz",
41+
"accuracy_tolerance": 0.3,
4142
"accuracy_metrics": {
42-
"fp32_mAP": 0.45252755065175254,
43-
"int8_mAP": 0.4561537594798616,
44-
"accuracy_drop": -0.00362620882
43+
"fp32_mAP": 0.4521154302569843,
44+
"int8_mAP": 0.4599022156047055,
45+
"accuracy_drop": -0.0077867853477212035
4546
},
4647
"performance_metrics": {
4748
"fp32_fps": 170.69,
@@ -88,19 +89,20 @@
8889
"backend": "onnx",
8990
"requirements": "examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/requirements.txt",
9091
"cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz",
92+
"accuracy_tolerance": 0.3,
9193
"accuracy_metrics": {
92-
"onnx_fp32_box_mAP": 0.44964819166629366,
93-
"onnx_fp32_mask_mAP": 0.36509415038708726,
94-
"onnx_int8_box_mAP": 0.45071569411799683,
95-
"onnx_int8_mask_mAP": 0.36805097120147484,
96-
"onnx_drop_box_mAP": -0.0010675024517031728,
97-
"onnx_drop_mask_mAP": -0.002956820814387584,
98-
"ov_fp32_box_mAP": 0.44964819083938,
94+
"onnx_fp32_box_mAP": 0.4496481927421265,
95+
"onnx_fp32_mask_mAP": 0.36509979887281135,
96+
"onnx_int8_box_mAP": 0.4509292170587964,
97+
"onnx_int8_mask_mAP": 0.36710935370534864,
98+
"onnx_drop_box_mAP": -0.001281024316669932,
99+
"onnx_drop_mask_mAP": -0.002009554832537286,
100+
"ov_fp32_box_mAP": 0.4496481927421265,
99101
"ov_fp32_mask_mAP": 0.36509979887281135,
100-
"ov_int8_box_mAP": 0.4462460126164857,
101-
"ov_int8_mask_mAP": 0.3631361727753598,
102-
"ov_drop_box_mAP": 0.003402178222894292,
103-
"ov_drop_mask_mAP": 0.001963626097451543
102+
"ov_int8_box_mAP": 0.4494792029705020,
103+
"ov_int8_mask_mAP": 0.3666179206494213,
104+
"ov_drop_box_mAP": 0.000168989771624439,
105+
"ov_drop_mask_mAP": -0.0015181217766099264
104106
},
105107
"performance_metrics": {
106108
"ov_fp32_fps": 139.32,
@@ -114,8 +116,8 @@
114116
"cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz",
115117
"accuracy_metrics": {
116118
"fp32_top1": 0.987770676612854,
117-
"int8_top1": 0.9775795936584473,
118-
"accuracy_drop": 0.010191082954406738
119+
"int8_top1": 0.9691720008850098,
120+
"accuracy_drop": 0.01859867572784424
119121
},
120122
"performance_metrics": {
121123
"fp32_fps": 1703.04,
@@ -226,11 +228,11 @@
226228
"python_version": [3,10,0],
227229
"accuracy_tolerance_after_training": 0.02,
228230
"accuracy_metrics": {
229-
"fp32_f1score": 0.9919999837875366,
231+
"fp32_f1score": 1.0,
230232
"int8_init_f1score": 0.9767441749572754
231233
},
232234
"accuracy_metrics_after_training": {
233-
"int8_f1score": 0.9919999837875366,
235+
"int8_f1score": 1.0,
234236
"accuracy_drop": 0.0
235237
},
236238
"performance_metrics": {
@@ -244,4 +246,4 @@
244246
"model_compression_rate": 3.7654144877995197
245247
}
246248
}
247-
}
249+
}

‎tests/cross_fw/examples/requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
-c ../../../constraints.txt
22
pytest
33
pytest-cov
4+
pytest-split

‎tests/cross_fw/examples/test_examples.py

+10-5
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ def test_examples(
5555
ov_version_override: str,
5656
data: str,
5757
):
58+
print("\n" + "-" * 64)
59+
print(f"Example name: {example_name}")
5860
python_version = sys.version_info
5961
example_python_version = tuple(example_params.get("python_version", python_version))
6062
if python_version < example_python_version:
@@ -63,19 +65,22 @@ def test_examples(
6365
backend = example_params["backend"]
6466
skip_if_backend_not_selected(backend, backends_list)
6567
venv_path = create_venv_with_nncf(tmp_path, "pip_e_local", "venv", {backend})
68+
pip_with_venv = get_pip_executable_with_venv(venv_path)
6669
if "requirements" in example_params:
67-
pip_with_venv = get_pip_executable_with_venv(venv_path)
6870
requirements = PROJECT_ROOT / example_params["requirements"]
6971
run_cmd_line = f"{pip_with_venv} install -r {requirements}"
7072
subprocess.run(run_cmd_line, check=True, shell=True)
7173

7274
if ov_version_override is not None:
73-
pip_with_venv = get_pip_executable_with_venv(venv_path)
7475
ov_version_cmd_line = f"{pip_with_venv} install {ov_version_override}"
7576
subprocess.run(ov_version_cmd_line, check=True, shell=True)
7677

78+
subprocess.run(f"{pip_with_venv} list", check=True, shell=True)
79+
7780
env = os.environ.copy()
7881
env["PYTHONPATH"] = str(PROJECT_ROOT) # need this to be able to import from tests.* in run_example.py
82+
env["ONEDNN_MAX_CPU_ISA"] = "AVX2" # Set ISA to AVX2 to get CPU independent results
83+
env["CUDA_VISIBLE_DEVICES"] = "" # Disable GPU
7984

8085
metrics_file_path = tmp_path / "metrics.json"
8186
python_executable_with_venv = get_python_executable_with_venv(venv_path)
@@ -87,17 +92,17 @@ def test_examples(
8792
cmd.run()
8893

8994
measured_metrics = load_json(metrics_file_path)
90-
95+
print(measured_metrics)
9196
for name, value in example_params[ACCURACY_METRICS].items():
9297
assert measured_metrics[name] == pytest.approx(
9398
value, abs=example_params.get("accuracy_tolerance", ACCURACY_TOLERANCE)
94-
)
99+
), f"metric {name}: {measured_metrics[name]} != {value}"
95100

96101
if ACCURACY_METRICS_AFTER_TRAINING in example_params:
97102
for name, value in example_params[ACCURACY_METRICS_AFTER_TRAINING].items():
98103
assert measured_metrics[name] == pytest.approx(
99104
value, abs=example_params.get("accuracy_tolerance_after_training", ACCURACY_TOLERANCE)
100-
)
105+
), f"metric {name}: {measured_metrics[name]} != {value}"
101106

102107
if MODEL_SIZE_METRICS in example_params:
103108
for name, value in example_params[MODEL_SIZE_METRICS].items():

0 commit comments

Comments
 (0)