opensearch-project · peterzhuamazon · Mar 4, 2025 · Mar 4, 2025 · Mar 4, 2025 · Mar 4, 2025
@@ -0,0 +1,78 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# This is a docker image specifically for assembling the DEB version of OpenSearch/OpenSearch-Dashboards
+# This is not capable of building k-NN plugin as it lacks the necessary old version of glibc on AL2
+
+FROM ubuntu:24.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
+
+# Remove ubuntu user which occupies the 1000 userid and groupid since 23.04
+# https://bugs.launchpad.net/cloud-images/+bug/2005129
+USER 0
+RUN touch /var/mail/ubuntu && chown ubuntu /var/mail/ubuntu && userdel -r ubuntu
+
+# Install python dependencies
+RUN apt-get update -y && apt-get upgrade -y && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa -y
+
+# Install necessary packages
+RUN apt-get update -y && apt-get install -y docker.io=24.0.7* curl build-essential git jq && \
+    apt-get install -y debmake debhelper-compat && \
+    apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libnss3 libxss1 xauth xvfb && \
+    apt-get install -y libxrender1 libxi6 libxtst6 libasound2t64 && \
+    apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev && \
+    apt-get install -y mandoc less && \
+    apt-get clean -y
+
+# Docker Compose v2
+RUN mkdir -p /usr/local/lib/docker/cli-plugins && \
+    if [ `uname -m` = "x86_64" ] || [ `uname -m` = "amd64" ]; then \
+        curl -SfL -o /usr/local/lib/docker/cli-plugins/docker-compose https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64; \
+    elif [ `uname -m` = "aarch64" ] || [ `uname -m` = "arm64" ]; then \
+        curl -SfL -o /usr/local/lib/docker/cli-plugins/docker-compose https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-aarch64; \
+    elif [ `uname -m` = "ppc64le" ]; then \
+        curl -SfL -o /usr/local/lib/docker/cli-plugins/docker-compose https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-ppc64le; \
+    else \
+        echo "Your system is not supported now" && exit 1; \
+    fi; \
+    chmod 755 /usr/local/lib/docker/cli-plugins/docker-compose && \
+    ln -s /usr/local/lib/docker/cli-plugins/docker-compose /usr/local/bin/docker-compose
+
+# Install python, update awscli to v2 due to lib conflicts on urllib3 v1 vs v2
+RUN apt-get update -y && apt-get install -y python3.9-full python3.9-dev && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 100 && \
+    update-alternatives --install /usr/bin/python python /usr/bin/python3.9 100 && \
+    update-alternatives --set python3 /usr/bin/python3.9 && \
+    update-alternatives --set python /usr/bin/python3.9 && \
+    curl -SL https://bootstrap.pypa.io/get-pip.py | python3 - && \
+    pip3 install awscliv2==2.3.1 pipenv==2023.6.12 && \
+    ln -s `which awsv2` /usr/local/bin/aws && aws --install
+
+# Install aptly and required changes to debmake
+# Remove lintian for now due to it takes nearly 20 minutes for OpenSearch as well as nearly an hour for OpenSearch-Dashboards during debmake
+RUN curl -SfL -o /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt && \
+    echo "deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release noble main" | tee -a /etc/apt/sources.list.d/aptly.list && \
+    apt-get update -y && apt-get install -y aptly=1.5.0* && apt-get clean -y && \
+    dpkg -r lintian
+
+# Tools setup
+COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/
+RUN apt-get install -y golang-1.22 && /tmp/jdk-setup.sh && /tmp/yq-setup.sh && /tmp/gh-setup.sh && apt-get clean -y && apt-get autoremove -y # Ubuntu has a bug where entrypoint=bash does not actually run .bashrc correctly
+
+# Create user group
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -s /bin/bash -d $CONTAINER_USER_HOME -m $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
+
+# Change User
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
+RUN aws --install
@@ -9,7 +9,8 @@
 # It has binfmt_support package installed to run non-native arch binary, as well as
 # qemu-user-static package to enable execution of different multi-arch containers
 
-# This can only be used on Ubuntu 2004 X64 version, as QEMU 5.0 is required to get buildx work properly without segfault
+# This can only be used on Ubuntu 2404 X64 version, as QEMU 8.2 is required to get buildx work properly without segfault
+# A similar issue on Ubuntu 2004 with QEMU 5.0 requirements has been documented here:
 # https://bugs.launchpad.net/ubuntu/+source/qemu/+bug/1928075
 
 # This image can be used with these arguments: -u root -v /var/run/docker.sock:/var/run/docker.sock
@@ -30,16 +31,18 @@ RUN apt-get update -y && apt-get install -y software-properties-common && add-ap
 
 # Install necessary packages to build multi-arch docker images
 RUN apt-get update -y && apt-get upgrade -y && apt-get install -y binfmt-support qemu-system qemu-system-common qemu-user qemu-user-static docker.io=24.0.7* curl && \
+    apt-get install -y mandoc less && \
     apt-get install -y debmake debhelper-compat
 
 # Install python, update awscli to v2 due to lib conflicts on urllib3 v1 vs v2
-RUN apt-get install -y python3.9-full && \
+RUN apt-get install -y python3.9-full python3.9-dev && \
     update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 100 && \
     update-alternatives --install /usr/bin/python python /usr/bin/python3.9 100 && \
     update-alternatives --set python3 /usr/bin/python3.9 && \
     update-alternatives --set python /usr/bin/python3.9 && \
     curl -SL https://bootstrap.pypa.io/get-pip.py | python3 - && \
-    pip3 install awscliv2==2.3.1
+    pip3 install awscliv2==2.3.1 && \
+    ln -s `which awsv2` /usr/local/bin/aws && aws --install
 
 # Install trivy to scan the docker images
 RUN apt-get install -y apt-transport-https gnupg lsb-release && \
@@ -59,6 +62,12 @@ RUN groupadd -g 1000 $CONTAINER_USER && \
     mkdir -p $CONTAINER_USER_HOME && \
     chown -R 1000:1000 $CONTAINER_USER_HOME
 
+# By default, awscliv2 will run with docker fallbacks and requires individual user to run `aws --install` to install binaries
+# https://pypi.org/project/awscliv2/
+USER $CONTAINER_USER
+RUN aws --install
+USER 0
+
 # ENV JDK
 ENV JAVA_HOME=/opt/java/openjdk-11
 ENV PATH=$PATH:$JAVA_HOME/bin

@@ -0,0 +1,190 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# This is a docker image specifically for setting up systemd env base for services with root user
+# It is initially designed to test pkg installation, but can be used for anything that requires systemd
+# It used the method posted by Daniel Walsh: https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container
+
+# In order to run images with systemd, you need to run in privileged mode: `docker run --privileged -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro <image_tag>`
+# If you use this image in jenkins pipeline you need to add these arguments: `args '--entrypoint=/usr/sbin/init -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro'`
+
+# 20230920: On Docker host with systemd version > 247 you need to use these args:
+# https://github.com/opensearch-project/opensearch-build/issues/4047
+# --entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host
+
+########################### Stage 0 ########################
+
+FROM ubuntu:24.04 AS linux_stage_0
+
+ENV container docker
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
+
+# Remove ubuntu user which occupies the 1000 userid and groupid since 23.04
+# https://bugs.launchpad.net/cloud-images/+bug/2005129
+USER 0
+RUN touch /var/mail/ubuntu && chown ubuntu /var/mail/ubuntu && userdel -r ubuntu
+
+SHELL ["/bin/bash", "-c"]
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install necessary packages
+RUN apt-get update -y && apt-get upgrade -y && apt-get install -y curl git gnupg2 tar procps build-essential cmake zip unzip jq && \
+    apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libnss3 libxss1 xauth xvfb && \
+    apt-get install -y libxrender1 libxi6 libxtst6 libasound2t64 && \
+    apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev && \
+    apt-get clean -y
+
+# Install yq
+COPY --chown=0:0 config/yq-setup.sh /tmp/
+RUN /tmp/yq-setup.sh
+
+# Create user group
+RUN groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -s /bin/bash -d $CONTAINER_USER_HOME -m $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME
+
+# Change User
+USER $CONTAINER_USER
+WORKDIR $CONTAINER_USER_HOME
+
+# Hard code node version and yarn version for now
+# nvm environment variables
+ENV NVM_DIR $CONTAINER_USER_HOME/.nvm
+ENV NODE_VERSION 20.18.3
+ENV CYPRESS_VERSION 12.13.0
+ARG CYPRESS_VERSION_LIST="5.6.0 9.5.4 12.13.0"
+ENV CYPRESS_LOCATION $CONTAINER_USER_HOME/.cache/Cypress/$CYPRESS_VERSION
+ENV CYPRESS_LOCATION_954 $CONTAINER_USER_HOME/.cache/Cypress/9.5.4
+# install nvm
+# https://github.com/creationix/nvm#install-script
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+# install node and npm
+RUN source $NVM_DIR/nvm.sh \
+    && nvm install $NODE_VERSION \
+    && nvm alias default $NODE_VERSION \
+    && nvm use default
+# add node and npm to path so the commands are available
+ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+# install yarn
+COPY --chown=$CONTAINER_USER:$CONTAINER_USER config/yarn-version.sh /tmp
+RUN npm install -g yarn@`/tmp/yarn-version.sh main`
+# Add legacy cypress@5.6.0 for 1.x line
+# Add legacy cypress@9.5.4 for pre-2.8.0 releases
+# Add latest cypress@12.13.0 for post-2.8.0 releases
+RUN for cypress_version in $CYPRESS_VERSION_LIST; do npm install -g cypress@$cypress_version && npm cache verify; done
+
+# Need root to get pass the build due to chrome sandbox needs to own by the root
+USER 0
+
+# Add legacy cypress 5.6.0 / 9.5.4 for ARM64 Architecture
+RUN if [ `uname -m` = "aarch64" ]; then for cypress_version in 5.6.0 9.5.4; do rm -rf $CONTAINER_USER_HOME/.cache/Cypress/$cypress_version && \
+    curl -SLO https://ci.opensearch.org/ci/dbc/tools/Cypress-$cypress_version-arm64.tar.gz && tar -xzf Cypress-$cypress_version-arm64.tar.gz -C $CONTAINER_USER_HOME/.cache/Cypress/ && \
+    chown $CONTAINER_USER:$CONTAINER_USER -R $CONTAINER_USER_HOME/.cache/Cypress/$cypress_version && rm -vf Cypress-$cypress_version-arm64.tar.gz; done; fi
+
+########################### Stage 1 ########################
+FROM ubuntu:24.04
+
+ARG CONTAINER_USER=ci-runner
+ARG CONTAINER_USER_HOME=/home/ci-runner
+
+# Remove ubuntu user which occupies the 1000 userid and groupid since 23.04
+# https://bugs.launchpad.net/cloud-images/+bug/2005129
+USER 0
+RUN touch /var/mail/ubuntu && chown ubuntu /var/mail/ubuntu && userdel -r ubuntu
+
+SHELL ["/bin/bash", "-c"]
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install python dependencies
+RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa -y
+
+# Install necessary packages
+RUN apt-get update -y && apt-get upgrade -y && apt-get install -y curl git gnupg2 tar procps build-essential cmake zip unzip jq && \
+    apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libnss3 libxss1 xauth xvfb && \
+    apt-get install -y libxrender1 libxi6 libxtst6 libasound2t64 && \
+    apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev && \
+    apt-get install -y mandoc less pigz && \
+    apt-get clean -y
+
+# Install python, update awscli to v2 due to lib conflicts on urllib3 v1 vs v2
+RUN apt-get update -y && apt-get install -y python3.9-full python3.9-dev && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 100 && \
+    update-alternatives --install /usr/bin/python python /usr/bin/python3.9 100 && \
+    update-alternatives --set python3 /usr/bin/python3.9 && \
+    update-alternatives --set python /usr/bin/python3.9 && \
+    curl -SL https://bootstrap.pypa.io/get-pip.py | python3 - && \
+    pip3 install awscliv2==2.3.1 pipenv==2023.6.12 cmake==3.26.4 && \
+    ln -s `which awsv2` /usr/local/bin/aws && aws --install
+
+# Create user group
+RUN apt-get install -y sudo && \
+    groupadd -g 1000 $CONTAINER_USER && \
+    useradd -u 1000 -g 1000 -s /bin/bash -d $CONTAINER_USER_HOME -m $CONTAINER_USER && \
+    mkdir -p $CONTAINER_USER_HOME && \
+    chown -R 1000:1000 $CONTAINER_USER_HOME && \
+    groupadd -g 1001 opensearch && \
+    useradd -u 1001 -g 1001 -s /bin/bash -d /home/opensearch -m opensearch && \
+    groupadd -g 1002 opensearch-dashboards && \
+    useradd -u 1002 -g 1002 -s /bin/bash -d /home/opensearch-dashboards -m opensearch-dashboards && \
+    usermod -a -G opensearch $CONTAINER_USER && \
+    usermod -a -G opensearch-dashboards $CONTAINER_USER && \
+    usermod -a -G adm $CONTAINER_USER && \
+    id && \
+    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which su`, `which usermod`, `which apt`, `which apt-get`, `which apt-key`, `which dpkg`, `which chmod`, `which kill`, `which curl`, `which tee`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
+
+# Copy from Stage0
+COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME
+ENV NVM_DIR $CONTAINER_USER_HOME/.nvm
+ENV NODE_VERSION 20.18.3
+ENV CYPRESS_VERSION 12.13.0
+ENV CYPRESS_LOCATION $CONTAINER_USER_HOME/.cache/Cypress/$CYPRESS_VERSION
+ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+
+# By default, awscliv2 will run with docker fallbacks and requires individual user to run `aws --install` to install binaries
+# https://pypi.org/project/awscliv2/
+USER $CONTAINER_USER
+RUN aws --install
+USER 0
+
+# Check dirs
+RUN source $NVM_DIR/nvm.sh && ls -al $CONTAINER_USER_HOME && echo $NODE_VERSION $NVM_DIR && nvm use $NODE_VERSION
+
+# Tools setup
+COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/
+RUN apt-get install -y golang-1.22 && /tmp/jdk-setup.sh && /tmp/yq-setup.sh && /tmp/gh-setup.sh && apt-get clean -y && apt-get autoremove -y
+
+# Setup Shared Memory
+RUN chmod -R 777 /dev/shm
+
+# We use the version test to check if packages installed correctly
+# And get added to the PATH
+# This will fail the docker build if any of the packages not exist
+RUN node -v
+RUN npm -v
+RUN yarn -v
+RUN cypress -v
+
+# Possible retain of multi-user.target.wants later due to PA
+# As of now we do not need this
+RUN apt-get -y install systemd procps util-linux openssl libssl-dev && apt-get clean -y && \
+    systemctl set-default multi-user && \
+(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+rm -f /lib/systemd/system/multi-user.target.wants/*;\
+rm -f /etc/systemd/system/*.wants/*;\
+rm -f /lib/systemd/system/local-fs.target.wants/*; \
+rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+rm -f /lib/systemd/system/basic.target.wants/*;\
+rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+CMD ["/usr/sbin/init"]
@@ -7,5 +7,16 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
-echo "Run systemd integTest for OpenSearch core engine"
-./gradlew qa:systemd-test:integTest --tests org.opensearch.systemdinteg.SystemdIT --console=plain
+set -e
+
+echo "Check if distribution is deb or rpm on linux"
+if [ "$OSTYPE" = "linux-gnu" ]; then
+    if (dpkg -s opensearch > /dev/null 2>&1) || (rpm -q opensearch > /dev/null 2>&1); then
+        echo "Run systemd integTest for OpenSearch core engine"
+        ./gradlew :qa:systemd-test:test --tests org.opensearch.systemdinteg.SystemdIntegTests --console=plain
+    else
+        echo "No deb or rpm installed detected, skip test"
+    fi
+else
+    echo "Not on linux host, skip test"
+fi
@@ -69,11 +69,11 @@ fi
 [ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m`
 
 ## Setup Performance Analyzer Agent
-mv $OUTPUT/plugins/opensearch-performance-analyzer/performance-analyzer-rca $OUTPUT/
+# mv $OUTPUT/plugins/opensearch-performance-analyzer/performance-analyzer-rca $OUTPUT/
 
 ## Performance Analyzer Configs
 if echo $ARTIFACTS | grep -Eo '/deb/|/rpm/'; then
     echo "DEB/RPM configs"
-    echo 'true' > $OUTPUT/../var/lib/opensearch/rca_enabled.conf
+    #echo 'true' > $OUTPUT/../var/lib/opensearch/rca_enabled.conf
     echo 'true' > $OUTPUT/../var/lib/opensearch/performance_analyzer_enabled.conf
 fi
@@ -8,7 +8,7 @@
 Source: opensearch-dashboards
 Section: web
 Priority: optional
-Maintainer: OpenSearch Team <opensearch@amazon.com>
+Maintainer: OpenSearch Team <release@opensearch.org>
 Build-Depends: debhelper-compat (= 12)
 Standards-Version: 4.5.0
 Homepage: https://opensearch.org/

@@ -1,6 +1,6 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: opensearch-dashboards
-Upstream-Contact: opensearch@amazon.com
+Upstream-Contact: release@opensearch.org
 Source: https://opensearch.org
 Files: *
 Copyright: OpenSearch Contributors

@@ -151,6 +151,6 @@ exit 0
 %{product_dir}/logs
 
 %changelog
-* Mon Mar 21 2022 OpenSearch Team <opensearch@amazon.com>
+* Mon Mar 21 2022 OpenSearch Team <release@opensearch.org>
 - Initial package
 
@@ -8,7 +8,7 @@
 Source: opensearch
 Section: web
 Priority: optional
-Maintainer: OpenSearch Team <opensearch@amazon.com>
+Maintainer: OpenSearch Team <release@opensearch.org>
 Build-Depends: debhelper-compat (= 12)
 Standards-Version: 4.5.0
 Homepage: https://opensearch.org/

@@ -1,6 +1,6 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: opensearch
-Upstream-Contact: opensearch@amazon.com
+Upstream-Contact: release@opensearch.org
 Source: https://opensearch.org
 Files: *
 Copyright: OpenSearch Contributors