Skip to content

Commit

Permalink
Add LibRdKafka to our CI (#2225)
Browse files Browse the repository at this point in the history
### Issues:
`CryptoAlg-2863`

### Description of changes: 
Add librdkafka upstream to our CI. This was added as a Github action to allow running the librdkafka tests. These tests require spinning up a kafka cluster which was running into issues in docker/codebuild. 

The patchfile in this PR is only to allow running a specific testing target in librdkafka. We do not need this patchfile for compatibility. 

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.
  • Loading branch information
smittals2 authored Mar 7, 2025
1 parent 4fb8ec3 commit 5054cf0
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 1 deletion.
16 changes: 15 additions & 1 deletion .github/workflows/integrations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,18 @@ jobs:
- uses: actions/checkout@v4
- name: Run libevent build
run: |
./tests/ci/integration/run_libevent_integration.sh
./tests/ci/integration/run_libevent_integration.sh
librdkafka:
if: github.repository_owner == 'aws'
runs-on: ubuntu-latest
steps:
- name: Install OS Dependencies
run: |
sudo apt-get update -o Acquire::Languages=none -o Acquire::Translation=none
sudo apt-get -y --no-install-recommends install \
cmake gcc ninja-build golang
- uses: actions/checkout@v4
- name: Run librdkafka build
run: |
./tests/ci/integration/run_librdkafka_integration.sh
13 changes: 13 additions & 0 deletions tests/ci/integration/librdkafka_patch/librdkafka-testing.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
diff --git a/tests/Makefile b/tests/Makefile
index 543639e4..41791162 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -51,7 +51,7 @@ full: broker broker_idempotent sasl
#
# The following targets require an existing cluster running (test.conf)
#
-quick:
+quick: $(BIN)
@echo "Running quick(er) test suite (without sockem)"
./run-test.sh -Q -E

77 changes: 77 additions & 0 deletions tests/ci/integration/run_librdkafka_integration.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0 OR ISC

set -ex

source tests/ci/common_posix_setup.sh

# Set up environment.

# SYS_ROOT
# - SRC_ROOT(aws-lc)
# - SCRATCH_FOLDER
# - KAFKA_SRC_FOLDER
# - AWS_LC_BUILD_FOLDER
# - AWS_LC_INSTALL_FOLDER

# Assumes script is executed from the root of aws-lc directory
SCRATCH_FOLDER="${SRC_ROOT}/KAFKA_BUILD_ROOT"
KAFKA_SRC_FOLDER="${SCRATCH_FOLDER}/librdkafka"
KAFKA_BUILD_PREFIX="${KAFKA_SRC_FOLDER}/build/install"
KAFKA_TEST_PATCH_FOLDER="${SRC_ROOT}/tests/ci/integration/librdkafka_patch"
AWS_LC_BUILD_FOLDER="${SCRATCH_FOLDER}/aws-lc-build"
AWS_LC_INSTALL_FOLDER="${SCRATCH_FOLDER}/aws-lc-install"

mkdir -p ${SCRATCH_FOLDER}
rm -rf "${SCRATCH_FOLDER:?}"/*
cd ${SCRATCH_FOLDER}

function kafka_build() {
export CFLAGS="-I${AWS_LC_INSTALL_FOLDER}/include ${CFLAGS}"
export CXXFLAGS="-I${AWS_LC_INSTALL_FOLDER}/include ${CXXFLAGS}"
export LDFLAGS="-L${AWS_LC_INSTALL_FOLDER}/lib ${LDFLAGS}"
export LD_LIBRARY_PATH="${AWS_LC_INSTALL_FOLDER}/lib"

./configure --prefix="$KAFKA_BUILD_PREFIX"
make -j install
make check

local kafka_executable="${KAFKA_BUILD_PREFIX}/lib/librdkafka.so"
ldd ${kafka_executable} \
| grep "${AWS_LC_INSTALL_FOLDER}/lib/libcrypto.so" || exit 1
}

function kafka_run_tests() {
export LD_LIBRARY_PATH="${AWS_LC_INSTALL_FOLDER}/lib"
python3 -m venv venv
source venv/bin/activate

pushd ${KAFKA_SRC_FOLDER}/tests
python3 -m pip install -U -r requirements.txt
python3 -m trivup.clusters.KafkaCluster --version 2.8.0 << EOF
TESTS_SKIP=0092,0113 make -j quick
exit
EOF
}

# This patch is only needed to execute the tests. A run_test executable is not
# available with the make quick target, this patch allows us to build that executable first.
function kafka_patch_test() {
patchfile="${KAFKA_TEST_PATCH_FOLDER}/librdkafka-testing.patch"
echo "Apply patch $patchfile..."
patch -p1 --quiet -i "$patchfile"
}

git clone https://github.com/confluentinc/librdkafka.git ${KAFKA_SRC_FOLDER}
mkdir -p ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER}
ls

aws_lc_build "$SRC_ROOT" "$AWS_LC_BUILD_FOLDER" "$AWS_LC_INSTALL_FOLDER" -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_TESTING=OFF -DBUILD_TOOL=OFF -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=1

# Build openvpn from source.
pushd ${KAFKA_SRC_FOLDER}
kafka_patch_test
kafka_build
kafka_run_tests
popd

0 comments on commit 5054cf0

Please sign in to comment.