More updates to the OSB build (opensearch-project#635)

Signed-off-by: Govind Kamat <govkamat@amazon.com>

Author: gkamat

.ci/build.sh

@@ -1,35 +1,31 @@
 #!/usr/bin/env bash
 
-pyenv_init() {
-  PATH=$HOME/.pyenv/shims:$PATH:$HOME/.pyenv/bin
-}
-
 function setup {
-  export THESPLOG_FILE="${THESPLOG_FILE:-${BENCHMARK_HOME}/.benchmark/logs/actor-system-internal.log}"
-  # this value is in bytes, the default is 50kB. We increase it to 200kiB.
-  export THESPLOG_FILE_MAXSIZE=${THESPLOG_FILE_MAXSIZE:-204800}
-  # adjust the default log level from WARNING
-  export THESPLOG_THRESHOLD="INFO"
+  export BENCHMARK_HOME=$GITHUB_WORKSPACE
+
+  export THESPLOG_FILE=$BENCHMARK_HOME/.benchmark/logs/actor-system-internal.log
+  export THESPLOG_FILE_MAXSIZE=204800			# default is 50 KiB
+  export THESPLOG_THRESHOLD=INFO			# default log level is WARNING
 
-  pyenv_init
   export TERM=dumb
   export LC_ALL=en_US.UTF-8
+
+  # Init pyenv.
+  PATH=$HOME/.pyenv/shims:$PATH:$HOME/.pyenv/bin
 }
 
-function build {
+function build_and_unit_test {
   setup
 
   set -e
-  make install-devel
+  make develop
   make lint
   make test
 }
 
-function build_it {
+function run_it {
   setup
 
-  export BENCHMARK_HOME="$GITHUB_WORKSPACE"
-
   docker pull ubuntu/squid:latest
 
   # make it38, it39, etc. so they run as concurrent GHA jobs

.github/workflows/integ-test.yml

@@ -42,6 +42,4 @@ jobs:
       - run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV
 
       - name: Run the CI build script
-        run: bash .ci/build.sh build_it ${{ matrix.python-version }}
-        env:
-          BENCHMARK_HOME: env.GITHUB_WORKSPACE
+        run: bash .ci/build.sh run_it ${{ matrix.python-version }}

.github/workflows/release-drafter.yml .github/workflows/publish-release.yml

@@ -1,12 +1,12 @@
-name: Release drafter
+name: Publish Release to GitHub
 
 on:
   push:
     tags:
       - "*"
 
 jobs:
-  draft-a-release:
+  publish-release:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
@@ -22,21 +22,20 @@ jobs:
           issue-title: 'Release opensearch-benchmark'
           issue-body: "Please approve or deny the release of opensearch-benchmark. **Tag**: ${{ github.ref_name }}  **Commit**: ${{ github.sha }}"
           exclude-workflow-initiator-as-approver: true
+
       - name: Set up Python 3
         uses: actions/setup-python@v3
         with:
           python-version: '3.x'
-      - name: Install build tools
-        run: |
-          python -m pip install --upgrade build
+
       - name: Build project for distribution
         run: |
-          python -m build
-          tar -zvcf artifacts.tar.gz dist
-      - name: Release
+          make build
+          tar zcvf artifacts.tar.gz dist
+
+      - name: Publish release
         uses: softprops/action-gh-release@v1
         with:
           draft: true
           generate_release_notes: true
-          files: |
-            artifacts.tar.gz
+          files: artifacts.tar.gz

.github/workflows/unit-test.yml

@@ -17,6 +17,4 @@ jobs:
         run: git clone https://github.com/pyenv/pyenv.git ~/.pyenv
 
       - name: Run the CI build script
-        run: bash .ci/build.sh build
-        env:
-          BENCHMARK_HOME: env.GITHUB_WORKSPACE
+        run: bash .ci/build.sh build_and_unit_test

DEVELOPER_GUIDE.md

@@ -23,16 +23,29 @@ This document will walk you through on what's needed to start contributing code
 
 ### Prerequisites
 
-  - **Pyenv**: Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding.
-    For more details please refer to the [PyEnv installation instructions](https://github.com/pyenv/pyenv#installation).
+  - **pyenv**: Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding.
+    For more details please refer to the [pyenv installation instructions](https://github.com/pyenv/pyenv#installation).
 
-    **Optional Step:** For Debian-based systems, install the following modules to continue with the next steps: 
+    `pyenv` requires that the C compiler and development libraries be installed, so that the specified Python versions can be build from source.  The installation instructions vary from platform to platform.
+
+    For Debian-based systems, install the following modules to continue with the next steps: 
     ```
     sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
     libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
     xz-utils tk-dev libffi-dev liblzma-dev git
     ```
 
+    For Amazon Linux 2023, run the following command:
+    ```
+    sudo yum -y install gcc openssl-devel bzip2-devel libffi-devel ncurses-devel sqlite-devel readline-devel zlib-devel xz-devel
+    ```
+
+    On the Mac platform, XCode needs to be installed as well as some additional required libraries:
+    ```
+    xcode-select --install
+    brew install pyenv jq zlib xz
+    ```
+
   - **JDK**: Although OSB is a Python application, it optionally builds and provisions OpenSearch clusters.  JDK version 17 is used to build the current version of OpenSearch.  Please refer to the [build setup requirements](https://github.com/opensearch-project/OpenSearch/blob/ca564fd04f5059cf9e3ce8aba442575afb3d99f1/DEVELOPER_GUIDE.md#install-prerequisites).
     Note that the `javadoc` executable should be available in the JDK installation.  An earlier version of the JDK can be used, but not all the integration tests will pass.
 
@@ -54,7 +67,7 @@ For those working on WSL2, it is recommended to clone the repository and set up
 After you git cloned the forked copy of OpenSearch Benchmark, use the following command-line instructions to set up OpenSearch Benchmark for development:
 ```
 cd opensearch-benchmark
-make install-devel
+make develop
 ```
 
 Depending on the platform and shell you have, use the following command to activate the virtual environment:

Makefile

@@ -16,11 +16,14 @@
 # under the License.
 
 SHELL = /bin/bash
+PYTHON = python3
 PIP = pip3
 VERSIONS = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d')
 VERSION38 = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d' | grep 3\.8)
 PYENV_ERROR = "\033[0;31mIMPORTANT\033[0m: Please install pyenv and run \033[0;31meval \"\$$(pyenv init -)\"\033[0m.\n"
 
+all: develop
+
 pyinst:
 	@which pyenv > /dev/null 2>&1 || { printf $(PYENV_ERROR); exit 1; }
 	@for i in $(VERSIONS); do pyenv install --skip-existing $$i; done
@@ -31,6 +34,10 @@ pyinst38:
 	pyenv install --skip-existing $(VERSION38)
 	pyenv local $(VERSION38)
 
+check-pip:
+	# Install pyenv if the Python environment is externally managed.
+	@if ! $(PIP) > /dev/null 2>&1 || ! $(PIP) install pip > /dev/null 2>&1; then make pyinst38; fi
+
 check-java:
 	@if ! test "$(JAVA_HOME)" || ! java --version > /dev/null 2>&1 || ! javadoc --help > /dev/null 2>&1; then \
 	    echo "Java installation issues for running integration tests" >&2; \
@@ -40,23 +47,22 @@ check-java:
 	    echo "NOTE: Java version 17 required to have all integration tests pass" >&2; \
 	fi
 
-install-deps: pyinst38
-	# @if test `uname` = Darwin -o `python3 --version | sed 's/.* 3.\([0-9]*\).*/3\1/'` -lt 38; then make pyinst38; fi
+install-deps: check-pip
 	$(PIP) install --upgrade pip setuptools wheel
 
-install-user: install-deps
-	PIP_ONLY_BINARY=h5py $(PIP) install -e .
+# pylint does not work with Python versions >3.8:
+#   Value 'Optional' is unsubscriptable (unsubscriptable-object)
+develop: pyinst38 install-deps
+	PIP_ONLY_BINARY=h5py $(PIP) install -e .[develop]
 
-install-devel: install-deps
-	$(PIP) install -e .[develop]
-
-wheel:
-	$(PIP) install --upgrade pip setuptools wheel
-	PIP_ONLY_BINARY=h5py $(PIP) wheel .
+build: install-deps
+	$(PIP) install --upgrade build
+	$(PYTHON) -m build
 
-install: wheel
-	PIP_ONLY_BINARY=h5py $(PIP) install opensearch_benchmark-*.whl
-	rm -r *.whl *.egg-info
+# Builds a wheel from source, then installs it.
+install: build
+	PIP_ONLY_BINARY=h5py $(PIP) install dist/opensearch_benchmark-*.whl
+	rm -rf dist
 
 clean:
 	rm -rf .benchmarks .eggs .tox .benchmark_it .cache build dist *.egg-info logs junit-py*.xml *.whl NOTICE.txt
@@ -75,7 +81,7 @@ tox-env-clean:
 lint:
 	@find osbenchmark benchmarks scripts tests it -name "*.py" -exec pylint -j0 -rn --load-plugins pylint_quotes --rcfile=$(CURDIR)/.pylintrc \{\} +
 
-test:
+test: develop
 	pytest tests/
 
 it: pyinst check-java python-caches-clean tox-env-clean

docker/Dockerfile

@@ -6,39 +6,41 @@
 # Stage 1: build packages and compile where needed
 #
 FROM python:3.11.2-slim AS build-stage
-WORKDIR /opensearch-benchmark
 
 RUN apt-get -y update && \
     apt-get install -y curl git gcc pbzip2 pigz make jq && \
     apt-get -y upgrade
 
 COPY . opensearch-benchmark
 
-RUN cd opensearch-benchmark; make wheel
+# There is no binary package currently available for yappi on ARM.
+RUN cd opensearch-benchmark; \
+    make build; \
+    if test "`uname -m`" = aarch64; then pip wheel yappi && cp yappi-*.whl dist; fi
 
 
 #
 # Stage 2: create image
 #
 FROM python:3.11.2-slim AS image-stage
-WORKDIR /opensearch-benchmark
 ENV BENCHMARK_RUNNING_IN_DOCKER=True
 
 RUN groupadd --gid 1000 opensearch-benchmark && \
     useradd -d /opensearch-benchmark -m -k /dev/null -g 1000 -N -u 1000 -l -s /bin/bash benchmark
 
+WORKDIR /opensearch-benchmark
+
 RUN mkdir -p /opensearch-benchmark/.benchmark && \
     chown -R 1000:0 /opensearch-benchmark/.benchmark
 
-COPY --from=build-stage /opensearch-benchmark/opensearch-benchmark/yappi-*.whl /opensearch-benchmark/opensearch-benchmark/opensearch_benchmark-*.whl ./
+COPY --from=build-stage /opensearch-benchmark/dist/*.whl ./
 
-# There is no binary package currently available for yappi on ARM.
 RUN set -ex; \
     apt-get -y update; \
     apt-get install -y git pbzip2; \
     apt-get -y upgrade; \
     rm -rf /var/lib/apt/lists/*; \
-    PIP_ONLY_BINARY=h5py pip install yappi-*.whl opensearch_benchmark-*.whl; \
+    PIP_ONLY_BINARY=h5py pip install *.whl; \
     rm *.whl
 
 USER 1000