From 401329930889cca10fa34bc066512bce091e1b2c Mon Sep 17 00:00:00 2001 From: Will Childs-Klein Date: Fri, 7 Jun 2024 10:12:31 -0400 Subject: [PATCH] Add integration tests for OpenSSL-linking 3p modules (#1587) This change expands our python integration tests to load and exercise libcrypto-linking 3rd party modules. Cases covered include statically linked AWS-LC (default AWS CRT) and dynamically linked OpenSSL (AWS CRT, PyOpenSSL, PyCA). We also test these integrations with python's AWS-LC built in FIPS mode. Due to the pre-release nature of python 3.13 and 3.14, there are a few caveats for later versions: 3.12 introduced [a change](https://github.com/python/cpython/issues/95299) to drop `setuptools` from default virtual environments, causing installation issues for PyCA and PyOpenSSL on 3.13+. To work around this, we allow installation failure for those dependencies on newer (currently both 3.13 and 3.14 are pre-release) and exit early from the relevant test cases. We also encountered some issues installing CRT bindings from source on 3.12+, so for those versions we install the pre-compiled wheel module from PyPI, which uses AWS-LC under the hood. --- .github/workflows/integrations.yml | 16 +- .../python_patch/3.13/aws-lc-cpython.patch | 262 ++++++++++++++++++ tests/ci/integration/python_tests/test_crt.py | 74 +++++ .../python_tests/test_cryptography.py | 21 ++ .../python_tests/test_pyopenssl.py | 17 ++ .../ci/integration/run_python_integration.sh | 97 ++++++- 6 files changed, 481 insertions(+), 6 deletions(-) create mode 100644 tests/ci/integration/python_patch/3.13/aws-lc-cpython.patch create mode 100644 tests/ci/integration/python_tests/test_crt.py create mode 100644 tests/ci/integration/python_tests/test_cryptography.py create mode 100644 tests/ci/integration/python_tests/test_pyopenssl.py diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index ab71319f95..b8d0b436c9 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -101,6 +101,7 @@ jobs: python-main: if: github.repository_owner == 'aws' runs-on: ubuntu-latest + name: Python main steps: - name: Install OS Dependencies run: | @@ -112,7 +113,17 @@ jobs: ./tests/ci/integration/run_python_integration.sh main python-releases: if: github.repository_owner == 'aws' + strategy: + fail-fast: false + matrix: + openssl_in_crt: + - "0" + - "1" + fips: + - "0" + - "1" runs-on: ubuntu-latest + name: Python releases (FIPS=${{ matrix.fips}} OPENSSL_IN_CRT=${{ matrix.openssl_in_crt }}) steps: - name: Install OS Dependencies run: | @@ -121,7 +132,10 @@ jobs: - uses: actions/checkout@v3 - name: Build AWS-LC, build python, run tests run: | - ./tests/ci/integration/run_python_integration.sh 3.10 3.11 3.12 + ./tests/ci/integration/run_python_integration.sh 3.10 3.11 3.12 3.13 + env: + FIPS: ${{ matrix.fips }} + AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: ${{ matrix.openssl_in_crt }} bind9: if: github.repository_owner == 'aws' runs-on: ubuntu-latest diff --git a/tests/ci/integration/python_patch/3.13/aws-lc-cpython.patch b/tests/ci/integration/python_patch/3.13/aws-lc-cpython.patch new file mode 100644 index 0000000000..45bf31533f --- /dev/null +++ b/tests/ci/integration/python_patch/3.13/aws-lc-cpython.patch @@ -0,0 +1,262 @@ +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py +index 6e63a88..7dc83d7 100644 +--- a/Lib/test/test_httplib.py ++++ b/Lib/test/test_httplib.py +@@ -2066,7 +2066,7 @@ def test_host_port(self): + + def test_tls13_pha(self): + import ssl +- if not ssl.HAS_TLSv1_3: ++ if not ssl.HAS_TLSv1_3 or "AWS-LC" in ssl.OPENSSL_VERSION: + self.skipTest('TLS 1.3 support required') + # just check status of PHA flag + h = client.HTTPSConnection('localhost', 443) +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index 0e50d09..f4b7b3c 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -41,6 +41,7 @@ + from ssl import Purpose, TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType + + Py_DEBUG_WIN32 = support.Py_DEBUG and sys.platform == 'win32' ++Py_OPENSSL_IS_AWSLC = "AWS-LC" in ssl.OPENSSL_VERSION + + PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) + HOST = socket_helper.HOST +@@ -174,7 +175,7 @@ def is_ubuntu(): + except FileNotFoundError: + return False + +-if is_ubuntu(): ++if is_ubuntu() and not Py_OPENSSL_IS_AWSLC: + def seclevel_workaround(*ctxs): + """"Lower security level to '1' and allow all ciphers for TLS 1.0/1""" + for ctx in ctxs: +@@ -4001,6 +4002,7 @@ def test_no_legacy_server_connect(self): + sni_name=hostname) + + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") ++ @unittest.skipIf(Py_OPENSSL_IS_AWSLC, "AWS-LC doesn't support (FF)DHE") + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + client_context, server_context, hostname = testing_context() +@@ -4364,14 +4366,14 @@ def test_session_handling(self): + def test_psk(self): + psk = bytes.fromhex('deadbeef') + +- client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ++ client_context, server_context, _ = testing_context() ++ + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(lambda hint: (None, psk)) + +- server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(lambda identity: psk) +@@ -4443,14 +4445,14 @@ def server_callback(identity): + self.assertEqual(identity, client_identity) + return psk + +- client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ++ client_context, server_context, _ = testing_context() ++ + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(client_callback) + +- server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(server_callback, identity_hint) +@@ -4461,7 +4463,10 @@ def server_callback(identity): + s.connect((HOST, server.port)) + + +-@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") ++@unittest.skipUnless( ++ has_tls_version('TLSv1_3') and not Py_OPENSSL_IS_AWSLC, ++ "Test needs TLS 1.3; AWS-LC doesn't support PHA" ++) + class TestPostHandshakeAuth(unittest.TestCase): + def test_pha_setter(self): + protocols = [ +@@ -4737,6 +4742,31 @@ def test_internal_chain_server(self): + self.assertEqual(res, b'\x02\n') + + ++@unittest.skipUnless(Py_OPENSSL_IS_AWSLC, "Only test this against AWS-LC") ++class TestPostHandshakeAuthAwsLc(unittest.TestCase): ++ def test_pha(self): ++ protocols = [ ++ ssl.PROTOCOL_TLS_SERVER, ssl.PROTOCOL_TLS_CLIENT ++ ] ++ for protocol in protocols: ++ client_ctx, server_ctx, hostname = testing_context() ++ client_ctx.load_cert_chain(SIGNED_CERTFILE) ++ self.assertEqual(client_ctx.post_handshake_auth, None) ++ with self.assertRaises(AttributeError): ++ client_ctx.post_handshake_auth = True ++ with self.assertRaises(AttributeError): ++ server_ctx.post_handshake_auth = True ++ ++ with ThreadedEchoServer(context=server_ctx) as server: ++ with client_ctx.wrap_socket( ++ socket.socket(), ++ server_hostname=hostname ++ ) as ssock: ++ ssock.connect((HOST, server.port)) ++ with self.assertRaises(NotImplementedError): ++ ssock.verify_client_post_handshake() ++ ++ + HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') + requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') +diff --git a/Modules/Setup b/Modules/Setup +index cd1cf24..53bcc4c 100644 +--- a/Modules/Setup ++++ b/Modules/Setup +@@ -208,11 +208,11 @@ PYTHONPATH=$(COREPYTHONPATH) + #_hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) -lcrypto + + # To statically link OpenSSL: +-# _ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ +-# -l:libssl.a -Wl,--exclude-libs,libssl.a \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a +-# _hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ ++ -l:libssl.a -Wl,--exclude-libs,libssl.a \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a + + # The _tkinter module. + # +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index f7fdbf4..204d501 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -187,6 +187,11 @@ extern const SSL_METHOD *TLSv1_2_method(void); + #endif + + ++#if defined(OPENSSL_NO_TLS_PHA) || !defined(TLS1_3_VERSION) || defined(OPENSSL_NO_TLS1_3) ++ #define PY_SSL_NO_POST_HS_AUTH ++#endif ++ ++ + enum py_ssl_error { + /* these mirror ssl.h */ + PY_SSL_ERROR_NONE, +@@ -231,7 +236,7 @@ enum py_proto_version { + PY_PROTO_TLSv1 = TLS1_VERSION, + PY_PROTO_TLSv1_1 = TLS1_1_VERSION, + PY_PROTO_TLSv1_2 = TLS1_2_VERSION, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) + PY_PROTO_TLSv1_3 = TLS1_3_VERSION, + #else + PY_PROTO_TLSv1_3 = 0x304, +@@ -293,7 +298,7 @@ typedef struct { + */ + unsigned int hostflags; + int protocol; +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int post_handshake_auth; + #endif + PyObject *msg_cb; +@@ -873,7 +878,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, + SSL_set_mode(self->ssl, + SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_AUTO_RETRY); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + if (sslctx->post_handshake_auth == 1) { + if (socket_type == PY_SSL_SERVER) { + /* bpo-37428: OpenSSL does not ignore SSL_VERIFY_POST_HANDSHAKE. +@@ -1016,6 +1021,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) + } while (err.ssl == SSL_ERROR_WANT_READ || + err.ssl == SSL_ERROR_WANT_WRITE); + Py_XDECREF(sock); ++ + if (ret < 1) + return PySSL_SetError(self, __FILE__, __LINE__); + if (PySSL_ChainExceptions(self) < 0) +@@ -2775,7 +2781,7 @@ static PyObject * + _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) + /*[clinic end generated code: output=532147f3b1341425 input=6bfa874810a3d889]*/ + { +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int err = SSL_verify_client_post_handshake(self->ssl); + if (err == 0) + return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); +@@ -3198,7 +3204,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); + X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + self->post_handshake_auth = 0; + SSL_CTX_set_post_handshake_auth(self->ctx, self->post_handshake_auth); + #endif +@@ -3576,7 +3582,7 @@ set_maximum_version(PySSLContext *self, PyObject *arg, void *c) + return set_min_max_proto_version(self, arg, 1); + } + +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + static PyObject * + get_num_tickets(PySSLContext *self, void *c) + { +@@ -3607,7 +3613,7 @@ set_num_tickets(PySSLContext *self, PyObject *arg, void *c) + + PyDoc_STRVAR(PySSLContext_num_tickets_doc, + "Control the number of TLSv1.3 session tickets"); +-#endif /* TLS1_3_VERSION */ ++#endif /* defined(TLS1_3_VERSION) */ + + static PyObject * + get_security_level(PySSLContext *self, void *c) +@@ -3710,14 +3716,14 @@ set_check_hostname(PySSLContext *self, PyObject *arg, void *c) + + static PyObject * + get_post_handshake_auth(PySSLContext *self, void *c) { +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + return PyBool_FromLong(self->post_handshake_auth); + #else + Py_RETURN_NONE; + #endif + } + +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + static int + set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { + if (arg == NULL) { +@@ -4959,14 +4965,14 @@ static PyGetSetDef context_getsetlist[] = { + (setter) _PySSLContext_set_msg_callback, NULL}, + {"sni_callback", (getter) get_sni_callback, + (setter) set_sni_callback, PySSLContext_sni_callback_doc}, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + {"num_tickets", (getter) get_num_tickets, + (setter) set_num_tickets, PySSLContext_num_tickets_doc}, + #endif + {"options", (getter) get_options, + (setter) set_options, NULL}, + {"post_handshake_auth", (getter) get_post_handshake_auth, +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + (setter) set_post_handshake_auth, + #else + NULL, diff --git a/tests/ci/integration/python_tests/test_crt.py b/tests/ci/integration/python_tests/test_crt.py new file mode 100644 index 0000000000..92e586f166 --- /dev/null +++ b/tests/ci/integration/python_tests/test_crt.py @@ -0,0 +1,74 @@ +import datetime + +import awscrt +import awscrt.auth +import awscrt.http +import boto3 +import botocore + +AWS_ACCESS_KEY_ID = "AKIDEXAMPLE" +AWS_SECRET_ACCESS_KEY = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" +AWS_SESSION_TOKEN = None + +AWS_REGION = "us-east-1" +SERVICE = "service" +DATE = datetime.datetime( + year=2015, + month=8, + day=30, + hour=12, + minute=36, + second=0, + tzinfo=datetime.timezone.utc, +) + +credentials_provider = awscrt.auth.AwsCredentialsProvider.new_static( + AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN +) + + +def test_awscrt_sigv4(): + signing_config = awscrt.auth.AwsSigningConfig( + algorithm=awscrt.auth.AwsSigningAlgorithm.V4, + signature_type=awscrt.auth.AwsSignatureType.HTTP_REQUEST_HEADERS, + credentials_provider=credentials_provider, + region=AWS_REGION, + service=SERVICE, + date=DATE, + ) + + http_request = awscrt.http.HttpRequest( + method="GET", + path="/", + headers=awscrt.http.HttpHeaders([("Host", "example.amazonaws.com")]), + ) + + signing_result = awscrt.auth.aws_sign_request(http_request, signing_config).result() + assert signing_result is not None + print(f"Authorization: {signing_result.headers.get('Authorization')}") + + +def test_boto3(): + # make an API call to exercise SigV4 request signing in the CRT. the creds are + # nonsense, but that's OK because we sign and make a request over the network + # to determine that. + client = boto3.client( + "s3", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + aws_session_token=AWS_SESSION_TOKEN, + ) + try: + client.list_buckets() + assert False, "ListBuckets succeeded when it shouldn't have" + except botocore.exceptions.ClientError as e: + # expect it to fail due to nonsense creds + assert "InvalidAccessKeyId" in e.response["Error"]["Code"] + + +if __name__ == "__main__": + # discover test functions defined in __main__'s scope and execute them. + for test in [test + "()" for test in globals() if test.startswith("test_")]: + print(f"running {test}...") + eval(test) + print("done.") diff --git a/tests/ci/integration/python_tests/test_cryptography.py b/tests/ci/integration/python_tests/test_cryptography.py new file mode 100644 index 0000000000..cd8b40b454 --- /dev/null +++ b/tests/ci/integration/python_tests/test_cryptography.py @@ -0,0 +1,21 @@ +import sys + +assert sys.version_info.major == 3, "Only python 3 supported" +if sys.version_info.minor >= 13: + print("Fernet import currently broken on python release candidates >= 3.13") + print("Returning early for now, need to check in on this post-release") + sys.exit() + +import cryptography +import cryptography.hazmat.backends.openssl.backend +from cryptography.fernet import Fernet + +# exercise simple round trip, then assert that PyCA has linked OpenSSL +k = Fernet.generate_key() +f = Fernet(k) +pt = b"hello world" +assert pt == f.decrypt(f.encrypt(pt)) + +version = cryptography.hazmat.backends.openssl.backend.openssl_version_text() +assert "OpenSSL" in version, f"PyCA didn't link OpenSSL: {version}" +assert "AWS-LC" not in version, f"PyCA linked AWS-LC: {version}" diff --git a/tests/ci/integration/python_tests/test_pyopenssl.py b/tests/ci/integration/python_tests/test_pyopenssl.py new file mode 100644 index 0000000000..d90cb1ca4e --- /dev/null +++ b/tests/ci/integration/python_tests/test_pyopenssl.py @@ -0,0 +1,17 @@ +import sys + +assert sys.version_info.major == 3, "Only python 3 supported" +if sys.version_info.minor >= 13: + print("PyOpenSSL import currently broken on mainline python >= 3.13.") + sys.exit() + +import OpenSSL +from OpenSSL import SSL + +# ensure libssl properly loaded +version = SSL.OpenSSL_version(SSL.OPENSSL_VERSION) +assert b"OpenSSL" in version, f"PyOpenSSL didn't link OpenSSL: {version}" +assert b"AWS-LC" not in version, f"PyOpenSSL linked AWS-LC: {version}" + +# ensure libcrypto properly loaded +assert len(OpenSSL.crypto.get_elliptic_curves()) > 0 diff --git a/tests/ci/integration/run_python_integration.sh b/tests/ci/integration/run_python_integration.sh index 49fb68430a..9b7ce3e726 100755 --- a/tests/ci/integration/run_python_integration.sh +++ b/tests/ci/integration/run_python_integration.sh @@ -10,6 +10,10 @@ set -exuo pipefail # Set up environment. +# Default env parameters to "off" +FIPS=${FIPS:-"0"} +AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=${AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO:-"0"} + # SYS_ROOT # - SRC_ROOT(aws-lc) # - SCRATCH_FOLDER @@ -24,8 +28,10 @@ set -exuo pipefail # Assumes script is executed from the root of aws-lc directory SCRATCH_FOLDER="${SRC_ROOT}/PYTHON_BUILD_ROOT" +CRT_SRC_FOLDER="${SCRATCH_FOLDER}/aws-crt-python" PYTHON_SRC_FOLDER="${SCRATCH_FOLDER}/python-src" PYTHON_PATCH_FOLDER="${SRC_ROOT}/tests/ci/integration/python_patch" +PYTHON_INTEG_TEST_FOLDER="${SRC_ROOT}/tests/ci/integration/python_tests" AWS_LC_BUILD_FOLDER="${SCRATCH_FOLDER}/aws-lc-build" AWS_LC_INSTALL_FOLDER="${SCRATCH_FOLDER}/aws-lc-install" @@ -42,10 +48,16 @@ function python_build() { function python_run_tests() { local branch=${1} + local python='./python' pushd ${branch} # We statically link, so need to call into python itself to assert that we're # correctly built against AWS-LC - ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + if [[ ${FIPS} == "1" ]]; then + local expected_version_str='AWS-LC FIPS' + else + local expected_version_str='AWS-LC' + fi + ${python} -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep "${expected_version_str}" # see https://github.com/pypa/setuptools/issues/3007 export SETUPTOOLS_USE_DISTUTILS=stdlib # A number of python module tests fail on our public CI images, but they're @@ -71,7 +83,80 @@ function python_run_tests() { popd } -# The per-branch patch files do a few things: +function fetch_crt_python() { + rm -rf ${CRT_SRC_FOLDER} + mkdir -p ${CRT_SRC_FOLDER} + pushd ${CRT_SRC_FOLDER} + git clone https://github.com/awslabs/aws-crt-python.git . + git submodule update --init + popd +} + +function install_crt_python() { + # for some reason on python 3.12+, AWS CRT's setup.py can't find + # setuptools/distutils installed to our virtualenv. for those versions, + # return early and let the CRT use precompiled PyPI wheel backed by AWS-LC. + python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep "AWS-LC" + if ! python -c 'import sys; assert sys.version_info.minor < 12, f"{sys.version_info}"'; then + python -m pip install awscrt + return + fi + python -m ensurepip + # setupttols not installed by default on more recent python versions + # see https://github.com/python/cpython/issues/95299 + python -m pip install setuptools wheel + python -m pip list + python -m pip install -e ${CRT_SRC_FOLDER} + # below was adapted from aws-crt-python's CI + # https://github.com/awslabs/aws-crt-python/blob/d76c3dacc94c1aa7dfc7346a77be78dc990b5171/.github/workflows/ci.yml#L159 + local awscrt_path=$(python -c "import _awscrt; print(_awscrt.__file__)") + echo "AWSCRT_PATH: $awscrt_path" + local linked_against=$(ldd $awscrt_path) + echo "LINKED AGAINST: $linked_against" + local uses_libcrypto_so=$(echo "$linked_against" | grep 'libcrypto*.so' | head -1) + echo "USES LIBCRYTPO: $uses_libcrypto_so" + # by default, the python CRT bindings bundle their own libcrypto wheel + # built from AWS-LC. AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO can be specified + # in build environment to tell CRT to link against system libcrypto + # (usually OpenSSL) instead. + if [[ ${AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO} == "1" ]]; then + test -n "$uses_libcrypto_so" + else + test -z "$uses_libcrypto_so" + fi +} + +function python_run_3rd_party_tests() { + local branch=${1} + pushd ${branch} + local venv=$(realpath '.venv') + echo creating virtualenv to isolate dependencies... + ./python -m virtualenv ${venv} || ./python -m venv ${venv} + source ${venv}/bin/activate + # assert that the virtual env's python is the binary we built w/ AWS-LC + which python | grep ${venv} + python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep "AWS-LC" + echo installing other OpenSSL-dependent modules... + install_crt_python + python -m pip install 'boto3[crt]' + # cffi install is busted on release candidates >= 3.13, so allow install + # failure for cryptography and pyopenssl on those versions for now. + python -m pip install 'cryptography' \ + || python -c 'import sys; assert sys.version_info.minor >= 3.13' + python -m pip install 'pyopenssl' \ + || python -c 'import sys; assert sys.version_info.minor >= 3.13' + echo running minor integration test of those dependencies... + for test in ${PYTHON_INTEG_TEST_FOLDER}/*.py; do + python ${test} + done + deactivate # function defined by .venv/bin/activate + rm -rf ${venv} + popd +} + +# The per-branch patch files do a few things for older versions (e.g. 3.10) +# that aren't taking non-security-critical patches (patches for newer versions +# likely only apply a subset of below): # # - Modify various unit tests to account for error string differences between # OpenSSL and AWS-LC. @@ -91,8 +176,6 @@ function python_run_tests() { # authentication portion of that protocol. # - Modify the ssl module's backing C code to account for AWS-LC's divergent # function signature and return value for |sk_SSL_CIPHER_find| -# -# TODO: Remove these patches when we make an upstream contribution. function python_patch() { local branch=${1} local src_dir="${PYTHON_SRC_FOLDER}/${branch}" @@ -125,7 +208,10 @@ mkdir -p ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER} aws_lc_build ${SRC_ROOT} ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER} \ -DBUILD_TESTING=OFF \ - -DBUILD_SHARED_LIBS=0 + -DBUILD_SHARED_LIBS=0 \ + -DFIPS=${FIPS} + +fetch_crt_python # Some systems install under "lib64" instead of "lib" ln -s ${AWS_LC_INSTALL_FOLDER}/lib64 ${AWS_LC_INSTALL_FOLDER}/lib @@ -142,6 +228,7 @@ for branch in "$@"; do python_patch ${branch} python_build ${branch} python_run_tests ${branch} + python_run_3rd_party_tests ${branch} done popd