Make compatible with requests 2.29.0 and urllib3 2.0 (ansible-collections#613)

* Make compatible with requests 2.29.0.

* This fix should also work with urllib3 2.0 according to urllib3 maintainer.

* Add changelog fragment.

* We still need the constraint for CI until Docker SDK for Python has a new release with a fix.

* Make modifications to response_class as small as possible.

* Revert "We still need the constraint for CI until Docker SDK for Python has a new release with a fix."

This reverts commit 698d544.

* The pip coming with the ansible-core 2.11 alpine3 image seems to be too old.

Author: felixfontein

.github/workflows/ansible-test.yml

@@ -101,6 +101,10 @@ jobs:
           - ''
         target:
           - ''
+        extra-constraints:
+          # Specifying this other than '' likely destroys change detection, but at least it will make
+          # CI pass when necessary...
+          - ''
         exclude:
           - ansible: ''
         include:
@@ -117,6 +121,7 @@ jobs:
             docker: alpine3
             python: ''
             target: azp/4/
+            extra-constraints: urllib3 < 2.0.0
           - ansible: '2.11'
             docker: alpine3
             python: ''
@@ -144,6 +149,9 @@ jobs:
             git clone --depth=1 --single-branch https://github.com/ansible-collections/community.crypto.git ../../community/crypto
             ;
             git clone --depth=1 --single-branch https://github.com/ansible-collections/community.general.git ../../community/general
+            ${{ matrix.extra-constraints && format('; echo ''{0}'' >> tests/utils/constraints.txt', matrix.extra-constraints) || '' }}
+            ;
+            cat tests/utils/constraints.txt
           pull-request-change-detection: 'true'
           target: ${{ matrix.target }}
           target-python-version: ${{ matrix.python }}

README.md

@@ -26,7 +26,7 @@ Some modules and plugins require Docker CLI, or other external, programs. Some r
 
 Installing the Docker SDK for Python also installs the requirements for the modules and plugins that use `requests`. If you want to directly install the Python libraries instead of the SDK, you need the following ones:
 
-- [requests](https://pypi.org/project/requests/) (versions before 2.29.0);
+- [requests](https://pypi.org/project/requests/);
 - [pywin32](https://pypi.org/project/pywin32/) when using named pipes on Windows with the Windows 32 API;
 - [paramiko](https://pypi.org/project/paramiko/) when using SSH to connect to the Docker daemon with `use_ssh_client=false`;
 - [pyOpenSSL](https://pypi.org/project/pyOpenSSL/) when using TLS to connect to the Docker daemon;

changelogs/fragments/613-requests.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - "Make vendored Docker SDK for Python code compatible with requests 2.29.0 and urllib3 2.0 (https://github.com/ansible-collections/community.docker/pull/613)."

meta/ee-requirements.txt

@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 docker
-urllib3 < 2.0  # TODO see https://github.com/ansible-collections/community.docker/issues/611
-requests < 2.29  # TODO see https://github.com/ansible-collections/community.docker/issues/611
+urllib3
+requests
 paramiko
 
 # We assume that EEs are not based on Windows, and have Python >= 3.5.

plugins/doc_fragments/docker.py

@@ -289,7 +289,7 @@ class ModuleDocFragment(object):
     communicate with the Docker daemon. It uses code derived from the Docker SDK or Python that is included in this
     collection.
 requirements:
-  - requests < 2.29.0 (see U(https://github.com/ansible-collections/community.docker/issues/611))
+  - requests
   - pywin32 (when using named pipes on Windows 32)
   - paramiko (when using SSH with I(use_ssh_client=false))
   - pyOpenSSL (when using TLS)

plugins/module_utils/_api/_import_helper.py

@@ -42,15 +42,20 @@ class InvalidSchema(Exception):
 
 try:
     from requests.packages import urllib3
+    from requests.packages.urllib3 import connection as urllib3_connection  # pylint: disable=unused-import
 except ImportError:
     try:
         import urllib3
+        from urllib3 import connection as urllib3_connection  # pylint: disable=unused-import
     except ImportError:
         URLLIB3_IMPORT_ERROR = traceback.format_exc()
 
         class _HTTPConnectionPool(object):
             pass
 
+        class _HTTPConnection(object):
+            pass
+
         class FakeURLLIB3(object):
             def __init__(self):
                 self._collections = self
@@ -63,7 +68,12 @@ def __init__(self):
                 self.match_hostname = object()
                 self.HTTPConnectionPool = _HTTPConnectionPool
 
+        class FakeURLLIB3Connection(object):
+            def __init__(self):
+                self.HTTPConnection = _HTTPConnection
+
         urllib3 = FakeURLLIB3()
+        urllib3_connection = FakeURLLIB3Connection()
 
 
 # Monkey-patching match_hostname with a version that supports

plugins/module_utils/_api/transport/npipeconn.py

@@ -10,24 +10,18 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from ansible.module_utils.six import PY3
 from ansible.module_utils.six.moves.queue import Empty
 
 from .. import constants
-from .._import_helper import HTTPAdapter, urllib3
+from .._import_helper import HTTPAdapter, urllib3, urllib3_connection
 
 from .basehttpadapter import BaseHTTPAdapter
 from .npipesocket import NpipeSocket
 
-if PY3:
-    import http.client as httplib
-else:
-    import httplib
-
 RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer
 
 
-class NpipeHTTPConnection(httplib.HTTPConnection, object):
+class NpipeHTTPConnection(urllib3_connection.HTTPConnection, object):
     def __init__(self, npipe_path, timeout=60):
         super(NpipeHTTPConnection, self).__init__(
             'localhost', timeout=timeout

plugins/module_utils/_api/transport/sshconn.py

@@ -24,12 +24,7 @@
 from .basehttpadapter import BaseHTTPAdapter
 from .. import constants
 
-if PY3:
-    import http.client as httplib
-else:
-    import httplib
-
-from .._import_helper import HTTPAdapter, urllib3
+from .._import_helper import HTTPAdapter, urllib3, urllib3_connection
 
 PARAMIKO_IMPORT_ERROR = None
 try:
@@ -120,7 +115,7 @@ def close(self):
         self.proc.terminate()
 
 
-class SSHConnection(httplib.HTTPConnection, object):
+class SSHConnection(urllib3_connection.HTTPConnection, object):
     def __init__(self, ssh_transport=None, timeout=60, host=None):
         super(SSHConnection, self).__init__(
             'localhost', timeout=timeout

plugins/module_utils/_api/transport/unixconn.py

@@ -13,29 +13,17 @@
 import socket
 
 from ansible.module_utils.six import PY2
-from ansible.module_utils.six.moves import http_client as httplib
 
 from .basehttpadapter import BaseHTTPAdapter
 from .. import constants
 
-from .._import_helper import HTTPAdapter, urllib3
+from .._import_helper import HTTPAdapter, urllib3, urllib3_connection
 
 
 RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer
 
 
-class UnixHTTPResponse(httplib.HTTPResponse, object):
-    def __init__(self, sock, *args, **kwargs):
-        disable_buffering = kwargs.pop('disable_buffering', False)
-        if PY2:
-            # FIXME: We may need to disable buffering on Py3 as well,
-            # but there's no clear way to do it at the moment. See:
-            # https://github.com/docker/docker-py/issues/1799
-            kwargs['buffering'] = not disable_buffering
-        super(UnixHTTPResponse, self).__init__(sock, *args, **kwargs)
-
-
-class UnixHTTPConnection(httplib.HTTPConnection, object):
+class UnixHTTPConnection(urllib3_connection.HTTPConnection, object):
 
     def __init__(self, base_url, unix_socket, timeout=60):
         super(UnixHTTPConnection, self).__init__(
@@ -58,10 +46,13 @@ def putheader(self, header, *values):
             self.disable_buffering = True
 
     def response_class(self, sock, *args, **kwargs):
-        if self.disable_buffering:
-            kwargs['disable_buffering'] = True
+        if PY2:
+            # FIXME: We may need to disable buffering on Py3 as well,
+            # but there's no clear way to do it at the moment. See:
+            # https://github.com/docker/docker-py/issues/1799
+            kwargs['buffering'] = not self.disable_buffering
 
-        return UnixHTTPResponse(sock, *args, **kwargs)
+        return super(UnixHTTPConnection, self).response_class(sock, *args, **kwargs)
 
 
 class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool):

tests/utils/constraints.txt

@@ -10,13 +10,11 @@ coverage >= 4.5.4, < 5.0.0 ; python_version > '3.7' # coverage had a bug in < 4.
 cryptography >= 1.3.0, < 2.2 ; python_version < '2.7' # cryptography 2.2 drops support for python 2.6
 cryptography >= 1.3.0, < 3.4 ; python_version < '3.6' # cryptography 3.4 drops support for python 2.7
 urllib3 < 1.24 ; python_version < '2.7' # urllib3 1.24 and later require python 2.7 or later
-urllib3 < 2.0.0 # TODO see https://github.com/ansible-collections/community.docker/issues/611
 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later
 paramiko < 2.4.0 ; python_version < '2.7' # paramiko 2.4.0 drops support for python 2.6
 paramiko < 3.0.0 ; python_version < '3.7' # paramiko 3.0.0 forces installation of a too new cryptography
 requests < 2.20.0 ; python_version < '2.7' # requests 2.20.0 drops support for python 2.6
 requests < 2.28 ; python_version < '3.7' # requests 2.28.0 drops support for python < 3.7
-requests < 2.29 # TODO see https://github.com/ansible-collections/community.docker/issues/611
 virtualenv < 16.0.0 ; python_version < '2.7' # virtualenv 16.0.0 and later require python 2.7 or later
 pyopenssl < 18.0.0 ; python_version < '2.7' # pyOpenSSL 18.0.0 and later require python 2.7 or later
 setuptools < 45 ; python_version <= '2.7' # setuptools 45 and later require python 3.5 or later