-
Notifications
You must be signed in to change notification settings - Fork 34
/
Copy pathaction.yml
175 lines (170 loc) · 7.19 KB
/
action.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
name: inductor-xpu-e2e-test
inputs:
suite:
required: true
type: string
default: 'huggingface'
description: Dynamo benchmarks test suite. huggingface,timm_models,torchbench. Delimiter is comma
env_prepare:
required: false
description: If set to any value, will prepare suite test env
dt:
required: true
type: string
default: 'float32'
description: Data precision of the test.float32,bfloat16,float16,amp_bf16,amp_fp16. Delimiter is comma
mode:
required: true
type: string
default: 'inference'
description: inference,training. Delimiter is comma
scenario:
required: true
type: string
default: 'accuracy'
description: accuracy,performance. Delimiter is comma
cards:
required: false
type: string
default: 'all'
description: which cards can be used in the test
hf_token:
required: false
description: HUGGING_FACE_HUB_TOKEN for torchbench test
pytorch:
required: false
type: string
default: 'main'
description: Pytorch branch/commit
driver:
required: false
type: string
default: 'lts'
description: Driver lts/rolling
runs:
using: composite
steps:
- name: Prepare ENV
if: ${{ inputs.env_prepare }}
shell: bash
run: |
pwd && printenv
source activate e2e_ci
source .github/scripts/env.sh ${{ inputs.pytorch }}
if [[ ${{ inputs.suite }} == *"torchbench"* ]]; then
if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
cd ../ && rm -rf audio && git clone --single-branch -b main https://github.com/pytorch/audio.git
cd audio && git checkout $TORCHAUDIO_COMMIT_ID
python setup.py bdist_wheel && pip uninstall torchaudio -y && pip install dist/*.whl
cd ../ && rm -rf vision && git clone --single-branch -b main https://github.com/pytorch/vision.git
cd vision && git checkout $TORCHVISION_COMMIT_ID
python setup.py bdist_wheel && pip uninstall torchvision -y && pip install dist/*.whl
fi
cd ../ && python -c "import torch, torchvision, torchaudio"
rm -rf benchmark && git clone https://github.com/pytorch/benchmark.git
cd benchmark && git checkout $TORCHBENCH_COMMIT_ID && pip install --no-deps -r requirements.txt
python install.py --continue_on_fail
# deps for torchrec_dlrm
pip install pyre_extensions
pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu
pip install torchmetrics==1.0.3
pip install torchrec --no-deps --index-url https://download.pytorch.org/whl/nightly/cpu
fi
if [[ ${{ inputs.suite }} == *"huggingface"* ]]; then
pip install --force-reinstall git+https://github.com/huggingface/transformers@${TRANSFORMERS_VERSION}
fi
if [[ ${{ inputs.suite }} == *"timm_models"* ]]; then
if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
cd ../ && rm -rf vision && git clone --single-branch -b main https://github.com/pytorch/vision.git
cd vision && git checkout $TORCHVISION_COMMIT_ID
python setup.py bdist_wheel && pip uninstall torchvision -y && pip install dist/*.whl
fi
# install timm without dependencies
pip install --no-deps git+https://github.com/huggingface/pytorch-image-models@$TIMM_COMMIT_ID
# install timm dependencies without torch and torchvision
pip install $(curl -sSL https://raw.githubusercontent.com/huggingface/pytorch-image-models/$TIMM_COMMIT_ID/requirements.txt | grep -vE torch)
fi
pip install numpy==1.26.4
- name: E2E Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }})
env:
HUGGING_FACE_HUB_TOKEN: ${{ inputs.hf_token }}
NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
shell: bash
run: |
source activate e2e_ci
source .github/scripts/env.sh ${{ inputs.pytorch }}
cp .github/scripts/inductor_xpu_test.sh ../pytorch
cd ../pytorch
# check param
function contains() {
contains_status="echo 'Start $2 ...'"
{
[[ $1 =~ (^|,)$2($|,) ]]
} || {
echo "[Warning] $2 is not suppotted type! Skipped!"
contains_status="continue"
}
}
set -xe
for suite in $(echo ${{ inputs.suite }} |sed 's/,/ /g')
do
contains "huggingface,timm_models,torchbench" $suite
$contains_status
for dt in $(echo ${{ inputs.dt }} |sed 's/,/ /g')
do
contains "float32,bfloat16,float16,amp_bf16,amp_fp16" $dt
$contains_status
for mode in $(echo ${{ inputs.mode }} |sed 's/,/ /g')
do
contains "inference,training" $mode
$contains_status
for scenario in $(echo ${{ inputs.scenario }} |sed 's/,/ /g')
do
contains "accuracy,performance" $scenario
$contains_status
if [ "${MODEL_ONLY_NAME}" == "" ];then
xpu_list=($(xpu-smi discovery |grep 'DRM Device: /dev/' |sed 's/.*card//;s/[^0-9].*//' |awk '{print $1 - 1":"NR - 1}'))
for xpu_id in ${xpu_list[*]}
do
bash inductor_xpu_test.sh ${suite} ${dt} ${mode} ${scenario} xpu ${xpu_id/:*} static ${#xpu_list[*]} ${xpu_id/*:} &
done
else
for test_model in $(echo ${MODEL_ONLY_NAME} |sed 's/,/ /g')
do
bash inductor_xpu_test.sh ${suite} ${dt} ${mode} ${scenario} xpu 0 static 1 0 ${test_model}
done
fi
wait
# summarize pass rate
LOG_DIR="inductor_log/${suite}/${dt}"
LOG_NAME=inductor_${suite}_${dt}_${mode}_xpu_${scenario}_all.log
rm -f ${LOG_DIR}/${LOG_NAME}
find ${LOG_DIR}/ -name "inductor_${suite}_${dt}_${mode}_xpu_${scenario}_card*.log" |xargs cat >> ${LOG_DIR}/${LOG_NAME} 2>&1
done
done
done
done
- name: Summary E2E Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }})
env:
HUGGING_FACE_HUB_TOKEN: ${{ inputs.hf_token }}
shell: bash
run: |
cd ../pytorch
rm -f inductor_log/summary_accuracy.csv
for var in $(find inductor_log/ -name "inductor_*_xpu_accuracy.csv")
do
sed -i "s/$/,$(basename $var)/" $var
cat $var >> inductor_log/summary_accuracy.csv
done
source activate e2e_ci
cd ${{ github.workspace }}
cp .github/scripts/inductor_summary.py ../pytorch
cd ../pytorch
pip install styleFrame scipy pandas
set -xe
dt=$(echo ${{ inputs.dt }} |sed 's/,/ /g')
mode=$(echo ${{ inputs.mode }} |sed 's/,/ /g')
suite=$(echo ${{ inputs.suite }} |sed 's/,/ /g')
scenario=$(echo ${{ inputs.scenario }} |sed 's/,/ /g')
python inductor_summary.py -p ${dt} -s ${suite} -m ${mode} -sc ${scenario}