Add LibRdKafka to our CI

.github/workflows/integrations.yml

@@ -34,4 +34,18 @@ jobs:
       - uses: actions/checkout@v4
       - name: Run libevent build
         run: |
-          ./tests/ci/integration/run_libevent_integration.sh
+          ./tests/ci/integration/run_libevent_integration.sh
+  librdkafka:
+    if: github.repository_owner == 'aws'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install OS Dependencies
+        run: |
+          sudo apt-get update -o Acquire::Languages=none -o Acquire::Translation=none
+          sudo apt-get -y --no-install-recommends install \
+          cmake gcc ninja-build golang
+      - uses: actions/checkout@v4
+      - name: Run librdkafka build
+        run: |
+          ./tests/ci/integration/run_librdkafka_integration.sh
+          

tests/ci/integration/librdkafka_patch/librdkafka-testing.patch

@@ -0,0 +1,13 @@
+diff --git a/tests/Makefile b/tests/Makefile
+index 543639e4..41791162 100644
+--- a/tests/Makefile
++++ b/tests/Makefile
+@@ -51,7 +51,7 @@ full: broker broker_idempotent sasl
+ #
+ # The following targets require an existing cluster running (test.conf)
+ #
+-quick:
++quick: $(BIN)
+ 	@echo "Running quick(er) test suite (without sockem)"
+ 	./run-test.sh -Q -E
+

tests/ci/integration/run_librdkafka_integration.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 OR ISC
+
+set -ex
+
+source tests/ci/common_posix_setup.sh
+
+# Set up environment.
+
+# SYS_ROOT
+#  - SRC_ROOT(aws-lc)
+#    - SCRATCH_FOLDER
+#      - KAFKA_SRC_FOLDER
+#      - AWS_LC_BUILD_FOLDER
+#      - AWS_LC_INSTALL_FOLDER
+
+# Assumes script is executed from the root of aws-lc directory
+SCRATCH_FOLDER="${SRC_ROOT}/KAFKA_BUILD_ROOT"
+KAFKA_SRC_FOLDER="${SCRATCH_FOLDER}/librdkafka"
+KAFKA_BUILD_PREFIX="${KAFKA_SRC_FOLDER}/build/install"
+KAFKA_TEST_PATCH_FOLDER="${SRC_ROOT}/tests/ci/integration/librdkafka_patch"
+AWS_LC_BUILD_FOLDER="${SCRATCH_FOLDER}/aws-lc-build"
+AWS_LC_INSTALL_FOLDER="${SCRATCH_FOLDER}/aws-lc-install"
+
+mkdir -p ${SCRATCH_FOLDER}
+rm -rf "${SCRATCH_FOLDER:?}"/*
+cd ${SCRATCH_FOLDER}
+
+function kafka_build() {
+  export CFLAGS="-I${AWS_LC_INSTALL_FOLDER}/include ${CFLAGS}"
+  export CXXFLAGS="-I${AWS_LC_INSTALL_FOLDER}/include ${CXXFLAGS}"
+  export LDFLAGS="-L${AWS_LC_INSTALL_FOLDER}/lib ${LDFLAGS}"
+  export LD_LIBRARY_PATH="${AWS_LC_INSTALL_FOLDER}/lib"
+
+  ./configure --prefix="$KAFKA_BUILD_PREFIX"
+  make -j install
+  make check
+
+  local kafka_executable="${KAFKA_BUILD_PREFIX}/lib/librdkafka.so"
+  ldd ${kafka_executable} \
+    | grep "${AWS_LC_INSTALL_FOLDER}/lib/libcrypto.so" || exit 1
+}
+
+function kafka_run_tests() {
+  export LD_LIBRARY_PATH="${AWS_LC_INSTALL_FOLDER}/lib"
+  python3 -m venv venv
+  source venv/bin/activate
+
+  pushd ${KAFKA_SRC_FOLDER}/tests
+  python3 -m pip install -U -r requirements.txt
+  python3 -m trivup.clusters.KafkaCluster --version 2.8.0 << EOF
+  TESTS_SKIP=0092,0113 make -j quick
+  exit
+EOF
+}
+
+# This patch is only needed to execute the tests. A run_test executable is not
+# available with the make quick target, this patch allows us to build that executable first.
+function kafka_patch_test() {
+  patchfile="${KAFKA_TEST_PATCH_FOLDER}/librdkafka-testing.patch"
+  echo "Apply patch $patchfile..."
+  patch -p1 --quiet -i "$patchfile"
+}
+
+git clone https://github.com/confluentinc/librdkafka.git ${KAFKA_SRC_FOLDER}
+mkdir -p ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER}
+ls
+
+aws_lc_build "$SRC_ROOT" "$AWS_LC_BUILD_FOLDER" "$AWS_LC_INSTALL_FOLDER" -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_TESTING=OFF -DBUILD_TOOL=OFF -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=1
+
+# Build openvpn from source.
+pushd ${KAFKA_SRC_FOLDER}
+kafka_patch_test
+kafka_build
+kafka_run_tests
+popd