Jenkinsfile

@NonCPS
List getMatrixAxes(Map matrix_axes) {
    List axes = []
    matrix_axes.each { axis, values ->
        List axisList = []
        values.each { value ->
            axisList << [(axis): value]
        }
        axes << axisList
    }
    // calculate cartesian product
    axes.combinations()*.sum()
}

IMAGE_NAME = "bert_${BUILD_NUMBER}"

// <matrix parameters initialized here>
Map compile_time_axes = [
    TF_VERSION:   ['1.15.4', '2.5.2', '2.9.0'],
    CXX_COMPILER: ['g++', 'icpx']
]

List ct_axes = getMatrixAxes(compile_time_axes).findAll { axis ->
    // NIGHTLY-only params
    !(params.NIGHTLY == null && axis['TF_VERSION'] in ['2.5.2'])
}

Map run_time_axes = [
    QUANTIZATION: ['--quantization', '--no-quantization'],
    BFLOAT16:     ['--no-bfloat16', '--bfloat16']
]

// TODO: remove when bf16 with quant is fixed
List rt_axes = getMatrixAxes(run_time_axes).findAll { axis ->
    !(axis['BFLOAT16'] == '--bfloat16' && axis['QUANTIZATION'] == '--quantization')
}

// </matrix parameters initialized here>

// <stages initialized here>
def prepareReports() {
    return {
        stage("Prepare Reports") {
            node(params.HOST) {
                dir("build") {
                    sh 'printf "Compiler\tModel\tTF\tQuantization\tBFloat16\tCorrect/Total\tResult\n" > accuracy.csv'
                    stash includes: 'accuracy.csv', name: 'accuracy'

                    sh 'printf "Compiler\tApp\tTF\tBERT variant\tQuantization\tBFloat16\tBatch Size\tThroughput\n" > benchmark.csv'
                    stash includes: 'benchmark.csv', name: 'benchmark'

                    sh 'printf "Compiler\tModel\tTF\tBERT variant\tQuantization\tBFloat16\tBatch Size\tResult\tThroughput\n" > model_zoo.csv'
                    stash includes: 'model_zoo.csv', name: 'model_zoo'

                    CPUS = sh (
                        script: 'lscpu | grep "NUMA node1" | sed "s/.*,//"',
                        returnStdout: true
                    ).trim() as String

                    env.NUMA = "numactl -C ${CPUS} --membind=1"
                }
            }
        }
    }
}

def publishReports() {
    return {
        stage("Publish Reports") {
            node(null) {
                checkout scm
                dir("build") {
                    sh "rm -rf nightly"
                    copyArtifacts filter: '*csv', projectName: 'main.bert/Nightly', selector: lastSuccessful(), target: 'nightly'
                    unstash 'accuracy'
                    unstash 'accuracy_pytorch'
                    unstash 'performance_pytorch'
                    unstash 'benchmark'
                    unstash 'model_zoo'
                    unstash 'bert_op_tests'

                    sh "python -m pip install --user -r ${WORKSPACE}/util/ci/requirements.txt"
                    sh "python ${WORKSPACE}/util/ci/compare_csv.py"

                    archiveArtifacts artifacts: "accuracy.csv"
                    archiveArtifacts artifacts: "accuracy_pytorch.csv"
                    archiveArtifacts artifacts: "performance_pytorch.csv"
                    archiveArtifacts artifacts: "benchmark.csv"
                    archiveArtifacts artifacts: "model_zoo.csv"
                    archiveArtifacts artifacts: "fullReport.html"
                    archiveArtifacts artifacts: 'test_results/*'

                    step([$class: 'ACIPluginPublisher', name: 'fullReport.html',  shownOnProjectPage: true])
                }
            }
        }
    }
}

def buildStages(List env) {
    return {
        withEnv(env) {
            stage("Build ${CXX_COMPILER} - ${TF_VERSION}") {
                dir('build') {
                    sh 'rm -rf CMakeCache.txt CMakeFiles *.csv *.html test_results'
                    sh """#!/bin/bash
                        set -e
                        source /opt/intel/oneapi/setvars.sh intel64

                        cmake --no-warn-unused-cli -DBACKENDS=TF \
                                                   -DPython3_EXECUTABLE:FILEPATH=/usr/bin/python \
                                                   -DCMAKE_C_COMPILER=icc \
                                                   -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
                                                   -DCMAKE_BUILD_TYPE:STRING=Release \
                                                   -G "Unix Makefiles" ..
                        cmake --build . --config Release --target clean -v -j --
                        cmake --build . --config Release --target all -v -j --
                    """
                }
            }
            
            if (TF_VERSION.startsWith('2'))
            stage("BertOp tests ${CXX_COMPILER} - ${TF_VERSION}") {
                dir('build') {
                    test_results = "bert_op_tests_${CXX_COMPILER}_tf${TF_VERSION}"
                    sh """#!/bin/bash
                        set -e
                        set -o pipefail # Ensure pytest exit code is forwarded through tee
                        source /opt/intel/oneapi/setvars.sh intel64
                        export BERT_OP_LIB=${WORKSPACE}/build/src/tf_op/libBertOp.so
                        export PYTHONPATH=\${PYTHONPATH}:${WORKSPACE}/python
                        mkdir -p test_results
                        python -m pytest --junitxml test_results/${test_results}.xml ${WORKSPACE}/tests/bert_op | tee test_results/${test_results}.log
                    """
                    stash includes: 'test_results/*', name: 'bert_op_tests'
                }
            }
        }
    }
}

def testStages(List env) {
    return {
        withEnv(env) {
            if (TF_VERSION.startsWith('2'))
            stage("benchmark ${CXX_COMPILER} - ${TF_VERSION} - ${QUANTIZATION} - ${BFLOAT16}") {
                dir('build') {
                    sh "echo in build: ${WORKSPACE} \$(pwd)"
                    unstash 'benchmark'
                    for(a in [10, 1000].findAll{x -> !(params.NIGHTLY == null && x==1000)}) {
                        for(BATCH_SIZE in [1, 32]) {
                            sh """#!/bin/bash
                                source /opt/intel/oneapi/setvars.sh intel64
                                export LD_PRELOAD=libjemalloc.so.2
                                ${NUMA} ./tests/benchmark/benchmark ${a} ${BATCH_SIZE} ${QUANTIZATION} ${BFLOAT16} | tee benchmark.log
                            """
                            sh "grep 'Average Throughput' benchmark.log | sed 's/.*Throughput: \\(.*\\)/${CXX_COMPILER}\\tbenchmark.exe ${a}\\t${TF_VERSION}\\tBERT-base\\t${QUANTIZATION}\\t${BFLOAT16}\\t${BATCH_SIZE}\\t\\1/' >> benchmark.csv"
                            sh """#!/bin/bash
                                source /opt/intel/oneapi/setvars.sh intel64
                                export LD_PRELOAD=libjemalloc.so.2
                                ${NUMA} ./tests/benchmark/benchmark ${a} ${BATCH_SIZE} ${QUANTIZATION} ${BFLOAT16} --large | tee benchmark.log
                            """
                            sh "grep 'Average Throughput' benchmark.log | sed 's/.*Throughput: \\(.*\\)/${CXX_COMPILER}\\tbenchmark.exe ${a}\\t${TF_VERSION}\\tBERT-large\\t${QUANTIZATION}\\t${BFLOAT16}\\t${BATCH_SIZE}\\t\\1/' >> benchmark.csv"
                        }
                    }
                    stash includes: 'benchmark.csv', name: 'benchmark'
                }
            }
            stage("accuracy ${CXX_COMPILER} - ${TF_VERSION} - ${QUANTIZATION} - ${BFLOAT16}"){
                dir('build') {
                    unstash 'accuracy'
                    sh """#!/bin/bash
                        export TF_VERSION=${TF_VERSION}
                        export TF_MAJOR=${TF_VERSION.replaceFirst(/(\d+)\..*/, '$1')}
                        export CXX_COMPILER=${CXX_COMPILER}
                        export QUANTIZATION=${QUANTIZATION}
                        export BFLOAT16=${BFLOAT16}
                        export path_to_bertop=${WORKSPACE}/build/src/tf_op/libBertOp.so   # accuracy_launcher.sh will pushd
                        export out_file=${WORKSPACE}/build/accuracy.csv                   # to its directory so use absolute
                        base_dir=/data/tf\${TF_MAJOR}                                     # paths to external resources
                        export Python3_EXECUTABLE=/usr/bin/python
                        export PYTHONPATH=\${PYTHONPATH}:${WORKSPACE}/python              # Need model modifier on PYTHONPATH
                        export TFDS_DATA_DIR=/data/tfds_data
                        export QUANT_FACTORS_DIR=${WORKSPACE}/jenkins_resources/tf\${TF_MAJOR}

                        source /opt/intel/oneapi/setvars.sh intel64
                        ${WORKSPACE}/tests/tf\${TF_MAJOR}_ops_accuracy/accuracy_launcher.sh \${base_dir}
                    """
                    stash includes: 'accuracy.csv', name: 'accuracy'
                }
            }
            if (TF_VERSION.startsWith('2'))
            stage("model-zoo ${CXX_COMPILER} - ${TF_VERSION} - ${QUANTIZATION} - ${BFLOAT16}"){
                dir('build') {
                    unstash 'model_zoo'
                    // BATCH_SIZE must divide 408 (dataset size) without a remainder
                    def QUANT_FILES = [
                        'BERT-base': 'quant_factors_uncased_L-12_H-768_A-12.txt',
                        'BERT-large': 'quant_factors_uncased_L-24_H-1024_A-16.txt']
                    def BASE_MODEL_DIRS = [
                        'BERT-base': '/data/model_zoo/bert-base-tf2',
                        'BERT-large': '/data/model_zoo/bert_large/uncased_L-24_H-1024_A-16'
                    ]
                    def ORIGINAL_MODEL_DIRS = [
                        'BERT-base': '/data/model_zoo/frozen',
                        'BERT-large': '/data/model_zoo/bert_large/frozen']
                    def MODIFIED_MODEL_DIRS = [
                        'BERT-base': '/data/model_zoo/frozen_modified',
                        'BERT-large': '/data/model_zoo/bert_large/frozen_modified']
                    for (MODEL in ['BERT-base', 'BERT-large']) {
                        for (BATCH_SIZE in [1, 24]) {
                            sh """#!/bin/bash
                                set -e
                                export PYTHONPATH=\${PYTHONPATH}:${WORKSPACE}/model_zoo/models/language_modeling/tensorflow/bert_large/inference
                                export PYTHONPATH=\${PYTHONPATH}:${WORKSPACE}/python
                                export TF_MAJOR=${TF_VERSION.replaceFirst(/(\d+)\..*/, '$1')}
                                export QUANT_FACTORS_PATH=${WORKSPACE}/jenkins_resources/tf\${TF_MAJOR}/${QUANT_FILES[MODEL]}
                                OUTPUT_DIR=\$(mktemp -d)
                                GLUE_DIR=/data/tf1/download_glue/glue_data
                                BERT_BASE_DIR=${BASE_MODEL_DIRS[MODEL]}
                                path_to_bertop=${WORKSPACE}/build/src/tf_op/libBertOp.so
                                source /opt/intel/oneapi/setvars.sh intel64
                                FROZEN_DIR=${ORIGINAL_MODEL_DIRS[MODEL]}
                                export LD_PRELOAD=libjemalloc.so.2

                                # running only once when both QUANTIZATION and BFLOAT16 are disabled
                                if [[ "${QUANTIZATION}" =~ "--no" && "${BFLOAT16}" =~ "--no" ]] ; then
                                    ${NUMA} python ${WORKSPACE}/tests/model_zoo/run_model_zoo.py \
                                    --frozen_graph_path=\${FROZEN_DIR}/frozen_model.pb \
                                    --output_dir=\${OUTPUT_DIR} \
                                    --bert_config_file=\${BERT_BASE_DIR}/bert_config.json \
                                    --do_train=False \
                                    --precision=fp32 \
                                    --do_lower_case=True \
                                    --task_name=MRPC \
                                    --do_eval=true \
                                    --vocab_file=\${BERT_BASE_DIR}/vocab.txt \
                                    --data_dir=\${GLUE_DIR}/MRPC \
                                    --eval_batch_size=${BATCH_SIZE} \
                                    --experimental_gelu=False \
                                    --max_seq_length=128 \
                                    --inter_op_parallelism_threads=1 \
                                    --intra_op_parallelism_threads=18 \
                                    \${path_to_bertop} 2>&1 | tee model_zoo.log

                                    model_zoo1=\$(grep -m1 'w:  eval_accuracy ' model_zoo.log | sed 's/.*= \\(.*\\)/'${CXX_COMPILER}'\tfrozen\t'${TF_VERSION}'\t'${MODEL}'\t'${QUANTIZATION}'\t'${BFLOAT16}'\t'${BATCH_SIZE}'\t\\1/' )
                                    model_zoo2=\$(grep -m1 'Throughput is ' model_zoo.log | sed 's/.*is \\(.*\\)/\\1/' )
                                    echo "\$model_zoo1\t\$model_zoo2" >> model_zoo.csv
                                fi

                                tmpdir=\$(mktemp -d)
                                cp -R ${MODIFIED_MODEL_DIRS[MODEL]}/* \${tmpdir}/
                                FROZEN_DIR=\${tmpdir}

                                python -m model_modifier.configure_bert_op \
                                    ${QUANTIZATION} \
                                    ${BFLOAT16} \
                                    --quant-factors-path=\${QUANT_FACTORS_PATH} \
                                    \${FROZEN_DIR}/frozen_graph.pb

                                ${NUMA} python ${WORKSPACE}/tests/model_zoo/run_model_zoo.py \
                                --frozen_graph_path=\${FROZEN_DIR}/frozen_graph.pb \
                                --output_dir=\${OUTPUT_DIR} \
                                --bert_config_file=\${BERT_BASE_DIR}/bert_config.json \
                                --do_train=False \
                                --precision=fp32 \
                                --do_lower_case=True \
                                --task_name=MRPC \
                                --do_eval=true \
                                --vocab_file=\${BERT_BASE_DIR}/vocab.txt \
                                --data_dir=\${GLUE_DIR}/MRPC \
                                --eval_batch_size=${BATCH_SIZE} \
                                --experimental_gelu=False \
                                --max_seq_length=128 \
                                --inter_op_parallelism_threads=1 \
                                --intra_op_parallelism_threads=18 \
                                \${path_to_bertop} 2>&1 | tee -a model_zoo.log

                                model_zoo3=\$(grep 'w:  eval_accuracy ' model_zoo.log | sed 's/.*= \\(.*\\)/'${CXX_COMPILER}'\tfrozen_modified\t'${TF_VERSION}'\t'${MODEL}'\t'${QUANTIZATION}'\t'${BFLOAT16}'\t'${BATCH_SIZE}'\t\\1/' | tail -1)
                                model_zoo4=\$(grep 'Throughput is ' model_zoo.log | sed 's/.*is \\(.*\\)/\\1/' | tail -1)
                                echo "\$model_zoo3\t\$model_zoo4" >> model_zoo.csv
                            """
                        }
                    }
                    stash includes: 'model_zoo.csv', name: 'model_zoo'
                }
            }
            if (TF_VERSION.startsWith('2'))
            stage("no-model-modifier ${CXX_COMPILER} - ${TF_VERSION} - ${QUANTIZATION} - ${BFLOAT16}"){
                // In this stage we only want to run a single iteration of the benchmark to verify that the
                // no-model-modifier approach is working. We don't need to repeat the TF performance runs.
                // We just `set -e` to make sure the stage fails
                dir('build') {
                    sh """#!/bin/bash
                        set -e
                        export PYTHONPATH=\${PYTHONPATH}:${WORKSPACE}/python
                        source /opt/intel/oneapi/setvars.sh intel64
                        export BERT_OP_LIB=${WORKSPACE}/build/src/tf_op/libBertOp.so
                        export QUANT_FACTORS_FILE=${WORKSPACE}/jenkins_resources/tf2/quant_factors_uncased_L-12_H-768_A-12.txt
                        ${WORKSPACE}/tests/tf2_no_model_modifier/benchmark.py \
                            -m bert-base-uncased \
                            --iterations 1 \
                            --bert-op \
                            ${QUANTIZATION} \
                            ${BFLOAT16} \
                            --quant-factors-path \${QUANT_FACTORS_FILE}

                    """
                }
            }
        }
    }
}

def requirementsForce(TF_VERSION)
{
    sh 'git reset --hard || echo "unable to reset repo"'
    sh 'git clean -ffdx || echo "unable to clean repo"'
    if (! TF_VERSION.startsWith('2.5.2')) {
        return
    }
    def TF_TEXT_VERSION = ['2.5.2': '2.5.0']
    sh "sed -r -i 's|(tensorflow-text)|\\1==${TF_TEXT_VERSION[TF_VERSION]}|' requirements-tf2.txt"
}

def buildDockerfileTensorflow(List TF_VERSION) {
    TF_VERSION.each { tf ->
    requirementsForce(tf)
    // tf 1.15 works with python3.6 only
    docker.build("${IMAGE_NAME}:${tf}", "--target=bert-op-tensorflow \
                                    --build-arg TF_VERSION=${tf} \
                                    --build-arg TF_requirements=requirements-tf${tf.replaceFirst(/(\d+)\..*/, '$1')}.txt \
                                    --build-arg PYTHON_VERSION=${tf.startsWith('1') ? "3.6" : "3.9"} \
                                    --build-arg username=\$(id -un)  --build-arg userid=\$(id -u) --build-arg usergroup=\$(id -g) \
                                    --build-arg http_proxy=\${http_proxy} \
                                    --build-arg https_proxy=\${https_proxy} \
                                    -f Dockerfile/Dockerfile-ci .")
    }
}


def pytorchStages() {
    return {
        stage("Build PyTorch Op") {
            dir('build') {
                sh 'rm -rf CMakeCache.txt CMakeFiles *.csv *.html test_results'
                sh """#!/bin/bash
                    set -e
                    source /opt/intel/oneapi/setvars.sh intel64

                    cmake --no-warn-unused-cli -DBACKENDS=PT \
                                                -DCMAKE_C_COMPILER=gcc \
                                                -DCMAKE_CXX_COMPILER=g++ \
                                                -DCMAKE_BUILD_TYPE:STRING=Release \
                                                -G "Unix Makefiles" ..
                    cmake --build . --config Release --target clean -v -j --
                    cmake --build . --config Release --target all -v -j --
                """
            }

        }
        stage("Accuracy PyTorch") {
            dir('build') {
                try {
                    unstash 'accuracy_pytorch'
                } catch(e) {
                    print "No accuracy_pytorch stash yet."
                }
                sh """#!/bin/bash
                    set -e
                    export PYTHONPATH=\${PYTHONPATH}:${WORKSPACE}/python
                    export BERT_OP_PT_LIB=${WORKSPACE}/build/src/pytorch_op/libBertOpPT.so
                    export HUGGINGFACE_HUB_CACHE=/data/huggingface_hub_cache
                    source /opt/intel/oneapi/setvars.sh intel64
                    ${WORKSPACE}/tests/pytorch/accuracy.py --output-csv accuracy_pytorch.csv -n 500
                """
                stash includes: 'accuracy_pytorch.csv', name: 'accuracy_pytorch'

            }
            
        }
    }
}

def pytorchStagesWithRuntimeAxes(List env) {
    return {
        withEnv(env) {
            stage("Performance PyTorch") {
                dir('build') {
                    try {
                        unstash 'performance_pytorch'
                    } catch(e) {
                        print "No performance_pytorch stash yet."
                    }
                    
                    for(batch in [1, 8]) {
                        for(model in ['bert-base-uncased', 'bert-large-uncased']) {
                            sh """#!/bin/bash
                                set -e
                                export PYTHONPATH=\${PYTHONPATH}:${WORKSPACE}/python
                                export BERT_OP_PT_LIB=${WORKSPACE}/build/src/pytorch_op/libBertOpPT.so
                                export HUGGINGFACE_HUB_CACHE=/data/huggingface_hub_cache
                                source /opt/intel/oneapi/setvars.sh intel64
                                
                                # Run the vanilla and IPEX models only for --no-quantization and --no-bfloat16
                                if [[ "${QUANTIZATION}" =~ "--no" && "${BFLOAT16}" =~ "--no" ]]; then
                                    ${WORKSPACE}/tests/pytorch/benchmark.py \
                                        --model ${model} \
                                        --batch-size ${batch} \
                                        --seq-len 128 \
                                        --run-time 30 \
                                        --output-csv performance_pytorch.csv
                                    
                                    ${WORKSPACE}/tests/pytorch/benchmark.py \
                                        --ipex \
                                        --model ${model} \
                                        --batch-size ${batch} \
                                        --seq-len 128 \
                                        --run-time 30 \
                                        --output-csv performance_pytorch.csv
                                fi

                                ${WORKSPACE}/tests/pytorch/benchmark.py \
                                    --bert-op \
                                    ${QUANTIZATION} \
                                    ${BFLOAT16} \
                                    --model ${model} \
                                    --batch-size ${batch} \
                                    --seq-len 128 \
                                    --run-time 30 \
                                    --output-csv performance_pytorch.csv
                            """
                        }
                    }
                    stash includes: 'performance_pytorch.csv', name: 'performance_pytorch'

                }
            }
        }
    }
}

def buildDockerfilePytorch()
{
    docker.build("${IMAGE_NAME}:pt", "--target=bert-op-pytorch \
                                    --build-arg username=\$(id -un)  --build-arg userid=\$(id -u) --build-arg usergroup=\$(id -g) \
                                    --build-arg http_proxy=\${http_proxy} \
                                    --build-arg https_proxy=\${https_proxy} \
                                    -f Dockerfile/Dockerfile-ci .")
}

// </stages initialized here>

// stages will not be aborted when one of them fails
Map tasks = [failFast: false]

for(Map ct_axis in ct_axes) {
    List ct_axisEnv = ct_axis.collect { k, v ->
        "${k}=${v}"
    }

    tasks[ct_axisEnv.join(', ')] = { ->
        withEnv(ct_axisEnv) {
            node(params.HOST) {
                checkout scm
                docker.image("${IMAGE_NAME}:${TF_VERSION}").inside("--privileged --init -v ${jenkins_data}:/data") {
                    buildStages(ct_axisEnv).call()
                    if (CXX_COMPILER == "icpx") {
                        for(Map rt_axis in rt_axes) {
                            List rt_axisEnv = rt_axis.collect { k, v ->
                                "${k}=${v}"
                            }
                            testStages(ct_axisEnv + rt_axisEnv).call()
                        }
                    }
                }
            }
        }
    }
}

tasks['PyTorch'] = { ->
    node(params.HOST) {
        checkout scm
        docker.image("${IMAGE_NAME}:pt").inside("--cpuset-cpus=0-10 --init -v ${jenkins_data}:/data") {
            pytorchStages().call()
            for(Map rt_axis in rt_axes) {
                List rt_axisEnv = rt_axis.collect { k, v ->
                    "${k}=${v}"
                }
                pytorchStagesWithRuntimeAxes(rt_axisEnv).call()
            }
        }
    }
}

stage("Matrix builds") {
    prepareReports().call()
    node(params.HOST) {
        checkout scm
        buildDockerfileTensorflow(ct_axes.collect{tf -> tf['TF_VERSION']}.unique())
        buildDockerfilePytorch()
    }
    parallel(tasks)
    publishReports().call()
}