From 836d4013d5470d21405ea82efcbe0cfd416d8743 Mon Sep 17 00:00:00 2001 From: Mika Laitio Date: Fri, 2 Aug 2024 17:01:27 -0700 Subject: [PATCH] fix the repository downloads - introduction of single repository download and build broke the download of all repositories... - problem was that the new code did not take into account the BINFO_APP_SRC_CLONE_DIR variable fixes: https://github.com/lamikr/rocm_sdk_builder/issues/120 Signed-off-by: Mika Laitio --- babs.sh | 35 +++--- binfo/core/003_llvm_project_llvm.binfo | 2 + build/binfo_utils.sh | 5 +- build/build_func.sh | 155 ++++++++++++++++--------- 4 files changed, 120 insertions(+), 77 deletions(-) diff --git a/babs.sh b/babs.sh index 1cb0223e..0d5214c9 100755 --- a/babs.sh +++ b/babs.sh @@ -160,10 +160,16 @@ func_repolist_upstream_remote_repo_add() { if [ ! -d src_projects ]; then echo "" echo "Download of source projects will start shortly" - echo "It will take up about 20 gb under 'src_projects' directory." - echo "Advice:" - echo "If you work with multible copies of this sdk," - echo "you could tar 'src_projects' and extract it manually for other SDK copies." + echo "Total download size under 'src_projects' directory will be about 30 GB." + echo "" + echo "----------------------------------------------------------------" + echo "You could speedup future builds by using a backup of src_projects directory:" + echo "Backup:" + echo " tar -cvf src_backup.tar 'src_projects'" + echo "Restore:" + echo " cd rocm_sdk_builder_dir2" + echo " tar -xvf src_backup.tar" + echo "----------------------------------------------------------------" echo "" sleep 3 fi @@ -171,7 +177,8 @@ func_repolist_upstream_remote_repo_add() { while [ "x${LIST_APP_SRC_CLONE_DIR[jj]}" != "x" ] do if [ ! -d ${LIST_APP_SRC_CLONE_DIR[$jj]} ]; then - echo "[${jj}]: Creating source code directory: ${LIST_APP_SRC_CLONE_DIR[$jj]}" + echo "" + echo "[${jj}]: Creating repository source code directory: ${LIST_APP_SRC_CLONE_DIR[$jj]}" sleep 0.1 mkdir -p ${LIST_APP_SRC_CLONE_DIR[$jj]} # LIST_APP_ADDED_UPSTREAM_REPO parameter is used in @@ -209,7 +216,7 @@ func_repolist_upstream_remote_repo_add() { # Fetch updates and initialize submodules while [ "x${LIST_APP_SRC_CLONE_DIR[jj]}" != "x" ] do - #echo "LIST_APP_ADDED_UPSTREAM_REPO[$jj]: ${LIST_APP_ADDED_UPSTREAM_REPO[$jj]}" + # echo "LIST_APP_ADDED_UPSTREAM_REPO[$jj]: ${LIST_APP_ADDED_UPSTREAM_REPO[$jj]}" # check if directory was just created and git fetch needs to be done if [ ${LIST_APP_ADDED_UPSTREAM_REPO[$jj]} -eq 1 ]; then echo "" @@ -248,7 +255,7 @@ func_repolist_upstream_remote_repo_add() { # apply patches if patch directory exists while [ "x${LIST_APP_PATCH_DIR[jj]}" != "x" ] do - #echo "LIST_APP_ADDED_UPSTREAM_REPO[$jj]: ${LIST_APP_ADDED_UPSTREAM_REPO[$jj]}" + # echo "LIST_APP_ADDED_UPSTREAM_REPO[$jj]: ${LIST_APP_ADDED_UPSTREAM_REPO[$jj]}" # check if directory was just created and git am needs to be done if [ ${LIST_APP_ADDED_UPSTREAM_REPO[$jj]} -eq 1 ]; then TEMP_PATCH_DIR=${LIST_APP_PATCH_DIR[$jj]} @@ -329,7 +336,8 @@ func_babs_init_and_fetch_single_repo_by_binfo() { # Initialize git repositories and add upstream remote if [ "x${BINFO_APP_SRC_DIR}" != "x" ]; then if [ ! -d ${BINFO_APP_SRC_DIR} ]; then - echo "Creating source code directory: ${BINFO_APP_SRC_DIR}" + echo "" + echo "Creating repository source code directory: ${BINFO_APP_SRC_DIR}" sleep 0.1 mkdir -p ${BINFO_APP_SRC_DIR} # LIST_APP_ADDED_UPSTREAM_REPO parameter is used in @@ -366,7 +374,7 @@ func_babs_init_and_fetch_single_repo_by_binfo() { #echo "CUR_APP_UPSTREAM_REPO_ADDED: ${CUR_APP_UPSTREAM_REPO_ADDED}" # check if directory was just created and git fetch needs to be done echo "" - echo "[${jj}]: Source Fetch" + echo "[${jj}]: Repository Source Code Fetch" echo "Repository name: ${BINFO_APP_NAME}" echo "Repository URL: ${BINFO_APP_UPSTREAM_REPO_URL}" echo "Source directory: ${BINFO_APP_SRC_DIR}" @@ -1131,15 +1139,6 @@ func_repolist_checkout_by_version_param() { fi } -#this method not used at the moment and needs refactoring if needed in future -func_repolist_download() { - func_build_version_init - func_envsetup_init - func_repolist_binfo_list_print - func_repolist_upstream_remote_repo_add - func_repolist_is_changes_committed -} - func_env_variables_print() { echo "SDK_CXX_COMPILER_DEFAULT: ${SDK_CXX_COMPILER_DEFAULT}" echo "HIP_PLATFORM_DEFAULT: ${HIP_PLATFORM_DEFAULT}" diff --git a/binfo/core/003_llvm_project_llvm.binfo b/binfo/core/003_llvm_project_llvm.binfo index 1251218a..73b2c063 100755 --- a/binfo/core/003_llvm_project_llvm.binfo +++ b/binfo/core/003_llvm_project_llvm.binfo @@ -1,6 +1,8 @@ BINFO_APP_NAME=llvm-project BINFO_APP_SRC_TOPDIR_BASENAME=${BINFO_APP_NAME} BINFO_APP_SRC_SUBDIR_BASENAME=llvm +# special case where the BINFO_APP_SRC_CLONE_DIR needs to be defined +# because it's different than the default location on BINFO_APP_SRC_DIR that is subfolder under it BINFO_APP_SRC_CLONE_DIR="${SDK_SRC_ROOT_DIR}/${BINFO_APP_SRC_TOPDIR_BASENAME}" BINFO_APP_SRC_DIR="${SDK_SRC_ROOT_DIR}/${BINFO_APP_SRC_TOPDIR_BASENAME}/${BINFO_APP_SRC_SUBDIR_BASENAME}" BINFO_APP_UPSTREAM_REPO_URL=https://github.com/ROCm/llvm-project diff --git a/build/binfo_utils.sh b/build/binfo_utils.sh index 05d36aa8..56d214e6 100644 --- a/build/binfo_utils.sh +++ b/build/binfo_utils.sh @@ -30,6 +30,9 @@ func_binfo_utils__init_binfo_app_list() { source "${LIST_BINFO_FILE_FULLNAME[jj]}" # Initialize source clone directory + # There are applications like llvm-project where the + # BINFO_APP_SRC_CLONE_DIR points to root and + # BINFO_APP_SRC_DIR points to llvm subfolder under it if [[ -n "${BINFO_APP_SRC_CLONE_DIR}" ]]; then LIST_APP_SRC_CLONE_DIR[jj]="${BINFO_APP_SRC_CLONE_DIR}" elif [[ -n "${BINFO_APP_SRC_DIR-}" ]]; then @@ -74,4 +77,4 @@ func_binfo_utils__init_binfo_app_list() { ((jj++)) done -} \ No newline at end of file +} diff --git a/build/build_func.sh b/build/build_func.sh index 2b301e4b..719ff074 100755 --- a/build/build_func.sh +++ b/build/build_func.sh @@ -10,16 +10,24 @@ func_upstream_remote_repo_add() { local CUR_APP_UPSTREAM_REPO_DEFINED=0 local CUR_APP_UPSTREAM_REPO_ADDED=0 + local ii + + if [[ -n "$1" ]]; then + ii=$1 + else + ii=1 + fi if [[ -n "${BINFO_APP_UPSTREAM_REPO_URL-}" ]]; then CUR_APP_UPSTREAM_REPO_DEFINED=1 fi # Initialize git repositories and add upstream remote - if [ "x${BINFO_APP_SRC_DIR}" != "x" ]; then - if [ ! -d ${BINFO_APP_SRC_DIR} ]; then - echo "Creating source code directory: ${BINFO_APP_SRC_DIR}" + if [ "x${BINFO_APP_SRC_CLONE_DIR}" != "x" ]; then + if [ ! -d ${BINFO_APP_SRC_CLONE_DIR} ]; then + echo "" + echo "[${ii}] Creating repository source code directory: ${BINFO_APP_SRC_CLONE_DIR}" sleep 0.1 - mkdir -p ${BINFO_APP_SRC_DIR} + mkdir -p ${BINFO_APP_SRC_CLONE_DIR} # LIST_APP_ADDED_UPSTREAM_REPO parameter is used in # situations where same src_code directory is used for building multiple projects # with just different configure parameters (for example amd-fftw) @@ -27,13 +35,13 @@ func_upstream_remote_repo_add() { CUR_APP_UPSTREAM_REPO_ADDED=1 fi if [ "$CUR_APP_UPSTREAM_REPO_DEFINED" == "1" ]; then - if [ ! -d ${BINFO_APP_SRC_DIR}/.git ]; then - cd "${BINFO_APP_SRC_DIR}" + if [ ! -d ${BINFO_APP_SRC_CLONE_DIR}/.git ]; then + cd "${BINFO_APP_SRC_CLONE_DIR}" echo "" - echo "Repository Init" + echo "[${ii}] Repository Init" echo "Repository name: $BINFO_APP_NAME}" echo "Repository URL: ${BINFO_APP_UPSTREAM_REPO_URL}" - echo "Source directory: ${BINFO_APP_SRC_DIR}" + echo "Source directory: ${BINFO_APP_SRC_CLONE_DIR}" echo "VERSION_TAG: ${BINFO_APP_UPSTREAM_REPO_VERSION_TAG}" sleep 0.5 git init @@ -42,11 +50,11 @@ func_upstream_remote_repo_add() { CUR_APP_UPSTREAM_REPO_ADDED=1 else CUR_APP_UPSTREAM_REPO_ADDED=0 - echo "${BINFO_APP_SRC_DIR} ok" + echo "${BINFO_APP_SRC_CLONE_DIR} ok" fi else CUR_APP_UPSTREAM_REPO_ADDED=0 - echo "${BINFO_APP_SRC_DIR} submodule ok" + echo "${BINFO_APP_SRC_CLONE_DIR} submodule ok" fi sleep 0.1 @@ -55,15 +63,15 @@ func_upstream_remote_repo_add() { # check if directory was just created and git fetch needs to be done if [ ${CUR_APP_UPSTREAM_REPO_ADDED} -eq 1 ]; then echo "" - echo "Source Fetch" + echo "[${ii}] Repository Source Code Fetch" echo "Repository name: ${BINFO_APP_NAME}" echo "Repository URL: ${BINFO_APP_UPSTREAM_REPO_URL}" - echo "Source directory: ${BINFO_APP_SRC_DIR}" + echo "Source directory: ${BINFO_APP_SRC_CLONE_DIR}" echo "VERSION_TAG: ${BINFO_APP_UPSTREAM_REPO_VERSION_TAG}" - cd "${BINFO_APP_SRC_DIR}" + cd "${BINFO_APP_SRC_CLONE_DIR}" git fetch upstream if [ $? -ne 0 ]; then - echo "git fetch failed: ${BINFO_APP_SRC_DIR}" + echo "git fetch failed: ${BINFO_APP_SRC_CLONE_DIR}" #exit 1 fi git fetch upstream --force --tags @@ -72,14 +80,14 @@ func_upstream_remote_repo_add() { ret_val=$? if [ ${ret_val} == "1" ]; then echo "" - echo "Submodule Init" + echo "[${ii}] Submodule Init" echo "Repository name: ${BINFO_APP_NAME}" echo "Repository URL: ${BINFO_APP_UPSTREAM_REPO_URL}" - echo "Source directory: ${BINFO_APP_SRC_DIR}" + echo "Source directory: ${BINFO_APP_SRC_CLONE_DIR}" echo "VERSION_TAG: ${BINFO_APP_UPSTREAM_REPO_VERSION_TAG}" git submodule update --init --recursive if [ $? -ne 0 ]; then - echo "git submodule init and update failed: ${BINFO_APP_SRC_DIR}" + echo "git submodule init and update failed: ${BINFO_APP_SRC_CLONE_DIR}" exit 1 fi fi @@ -90,14 +98,14 @@ func_upstream_remote_repo_add() { # check if directory was just created and git am needs to be done if [ ${CUR_APP_UPSTREAM_REPO_ADDED} -eq 1 ]; then local TEMP_PATCH_DIR=${CUR_APP_PATCH_DIR} - cd "${BINFO_APP_SRC_DIR}" + cd "${BINFO_APP_SRC_CLONE_DIR}" if [ -d "${TEMP_PATCH_DIR}" ]; then if [ ! -z "$(ls -A $TEMP_PATCH_DIR)" ]; then echo "" - echo "Applying Patches" + echo "[${ii}] Applying Patches" echo "Repository name: ${BINFO_APP_NAME}" echo "Repository URL: ${BINFO_APP_UPSTREAM_REPO_URL}" - echo "Source directory: ${BINFO_APP_SRC_DIR}" + echo "Source directory: ${BINFO_APP_SRC_CLONE_DIR}" echo "VERSION_TAG: ${BINFO_APP_UPSTREAM_REPO_VERSION_TAG}" echo "Patch dir: ${TEMP_PATCH_DIR}" git am --keep-cr "${TEMP_PATCH_DIR}"/*.patch @@ -107,13 +115,13 @@ func_upstream_remote_repo_add() { echo "Error, failed to Apply Patches" echo "Repository name: ${BINFO_APP_NAME}" echo "Repository URL: ${BINFO_APP_UPSTREAM_REPO_URL}" - echo "Source directory: ${BINFO_APP_SRC_DIR}" + echo "Source directory: ${BINFO_APP_SRC_CLONE_DIR}" echo "Version tag: ${BINFO_APP_UPSTREAM_REPO_VERSION_TAG}" echo "Patch dir: ${TEMP_PATCH_DIR}" echo "" exit 1 else - echo "Patches Applied: ${BINFO_APP_SRC_DIR}" + echo "Patches Applied: ${BINFO_APP_SRC_CLONE_DIR}" fi else echo "Warning, patch directory exists but is empty: ${TEMP_PATCH_DIR}" @@ -131,40 +139,57 @@ func_upstream_remote_repo_add() { } func_upstream_remote_repo_add_by_binfo() { - APP_INFO_FULL_NAME=$1 + local ii + + APP_INFO_FULL_NAME=$1 + if [[ -n "$2" ]]; then + ii=$2 + else + ii=1 + fi + echo "APP_INFO_FULL_NAME: ${APP_INFO_FULL_NAME}" - - #set build and install array commands that can be used for overriding default behavior to empty - unset BINFO_APP_CMAKE_CFG + #set build and install array commands that can be used for overriding default behavior to empty + unset BINFO_APP_CMAKE_CFG unset BINFO_APP_PRE_CONFIG_CMD_ARRAY - unset BINFO_APP_CONFIG_CMD_ARRAY - unset BINFO_APP_POST_CONFIG_CMD_ARRAY - unset BINFO_APP_BUILD_CMD_ARRAY - unset BINFO_APP_INSTALL_CMD_ARRAY - unset BINFO_APP_POST_INSTALL_CMD_ARRAY - unset BINFO_APP_CMAKE_BUILD_TYPE + unset BINFO_APP_CONFIG_CMD_ARRAY + unset BINFO_APP_POST_CONFIG_CMD_ARRAY + unset BINFO_APP_BUILD_CMD_ARRAY + unset BINFO_APP_INSTALL_CMD_ARRAY + unset BINFO_APP_POST_INSTALL_CMD_ARRAY + unset BINFO_APP_CMAKE_BUILD_TYPE - unset BINFO_APP_NO_PRECONFIG - unset BINFO_APP_NO_CONFIG - unset BINFO_APP_NO_POSTCONFIG - unset BINFO_APP_NO_BUILD - unset BINFO_APP_NO_INSTALL - unset BINFO_APP_NO_POSTINSTALL - unset BINFO_APP_NO_BUILD_CMD_RESULT_CHECK - unset BINFO_APP_NO_INSTALL_CMD_RESULT_CHECK - unset BINFO_APP_PRE_CONFIG_CMD_EXECUTE_ALWAYS - unset BINFO_APP_CONFIG_CMD_EXECUTE_ALWAYS - unset BINFO_APP_POST_CONFIG_CMD_EXECUTE_ALWAYS - unset BINFO_APP_POST_INSTALL_CMD_EXECUTE_ALWAYS + unset BINFO_APP_NO_PRECONFIG + unset BINFO_APP_NO_CONFIG + unset BINFO_APP_NO_POSTCONFIG + unset BINFO_APP_NO_BUILD + unset BINFO_APP_NO_INSTALL + unset BINFO_APP_NO_POSTINSTALL + unset BINFO_APP_NO_BUILD_CMD_RESULT_CHECK + unset BINFO_APP_NO_INSTALL_CMD_RESULT_CHECK + unset BINFO_APP_PRE_CONFIG_CMD_EXECUTE_ALWAYS + unset BINFO_APP_CONFIG_CMD_EXECUTE_ALWAYS + unset BINFO_APP_POST_CONFIG_CMD_EXECUTE_ALWAYS + unset BINFO_APP_POST_INSTALL_CMD_EXECUTE_ALWAYS unset BINFO_APP_UPSTREAM_REPO_VERSION_TAG - #read config and build commands - source ${APP_INFO_FULL_NAME} - res=$? - if [ ! $res -eq 0 ]; then - echo "failed to execute build env script: ${APP_INFO_FULL_NAME}" - exit 1 - fi + #read config and build commands + source ${APP_INFO_FULL_NAME} + res=$? + if [ ! $res -eq 0 ]; then + echo "failed to execute build env script: ${APP_INFO_FULL_NAME}" + exit 1 + fi + + # Initialize BINFO_APP_SRC_CLONE_DIR variable if it is not specified in the binfo file + if [[ ! -n "${BINFO_APP_SRC_CLONE_DIR}" ]]; then + if [[ -n "${BINFO_APP_SRC_DIR-}" ]]; then + BINFO_APP_SRC_CLONE_DIR="${BINFO_APP_SRC_DIR}" + else + echo "Error, you must define either the BINFO_APP_SRC_DIR or BINFO_APP_SRC_CLONE_DIR in ${APP_INFO_FULL_NAME}" + exit 1 + fi + fi # Initialize upstream repository version tag if [[ -n "${BINFO_APP_UPSTREAM_REPO_VERSION_TAG-}" ]]; then @@ -172,8 +197,8 @@ func_upstream_remote_repo_add_by_binfo() { else BINFO_APP_UPSTREAM_REPO_VERSION_TAG="${UPSTREAM_REPO_VERSION_TAG_DEFAULT}" fi - if [ ! -d ${BINFO_APP_SRC_DIR} ]; then - func_upstream_remote_repo_add + if [ ! -d ${BINFO_APP_SRC_CLONE_DIR} ]; then + func_upstream_remote_repo_add ${ii} fi } @@ -206,17 +231,20 @@ func_build_env_deinit() { } func_build_single_binfo() { + local ii; APP_INFO_FULL_NAME=$1 - echo "APP_INFO_FULL_NAME: ${APP_INFO_FULL_NAME}" - + if [[ -n "$2" ]]; then ii=$2 else ii=1 fi - + + echo "APP_INFO_FULL_NAME: ${APP_INFO_FULL_NAME}" #set build and install array commands that can be used for overriding default behavior to empty unset BINFO_APP_CMAKE_CFG + unset BINFO_APP_SRC_DIR + unset BINFO_APP_SRC_CLONE_DIR unset BINFO_APP_PRE_CONFIG_CMD_ARRAY unset BINFO_APP_CONFIG_CMD_ARRAY unset BINFO_APP_POST_CONFIG_CMD_ARRAY @@ -265,6 +293,16 @@ func_build_single_binfo() { exit 1 fi + # Initialize BINFO_APP_SRC_CLONE_DIR variable if it is not specified in the binfo file + if [[ ! -n "${BINFO_APP_SRC_CLONE_DIR}" ]]; then + if [[ -n "${BINFO_APP_SRC_DIR-}" ]]; then + BINFO_APP_SRC_CLONE_DIR="${BINFO_APP_SRC_DIR}" + else + echo "Error, you must define either the BINFO_APP_SRC_DIR or BINFO_APP_SRC_CLONE_DIR in ${APP_INFO_FULL_NAME}" + exit 1 + fi + fi + # Initialize upstream repository version tag if [[ -n "${BINFO_APP_UPSTREAM_REPO_VERSION_TAG-}" ]]; then BINFO_APP_UPSTREAM_REPO_VERSION_TAG="${BINFO_APP_UPSTREAM_REPO_VERSION_TAG}" @@ -286,6 +324,7 @@ func_build_single_binfo() { echo "BINFO_APP_SRC_SUBDIR_BASENAME: ${BINFO_APP_SRC_SUBDIR_BASENAME}" echo "BINFO_APP_SRC_TOPDIR_BASENAME: ${BINFO_APP_SRC_TOPDIR_BASENAME}" echo "BINFO_APP_SRC_DIR: ${BINFO_APP_SRC_DIR}" + echo "BINFO_APP_SRC_CLONE_DIR: ${BINFO_APP_SRC_CLONE_DIR}" echo "BINFO_APP_BUILD_DIR: ${BINFO_APP_BUILD_DIR}" echo "HIP_PATH: ${HIP_PATH}" echo "INSTALL_DIR: ${INSTALL_DIR_PREFIX_SDK_ROOT}" @@ -295,8 +334,8 @@ func_build_single_binfo() { echo "---------------------------" echo "" - if [ ! -d ${BINFO_APP_SRC_DIR} ]; then - func_upstream_remote_repo_add + if [ ! -d ${BINFO_APP_SRC_CLONE_DIR} ]; then + func_upstream_remote_repo_add ${ii} fi res=0