Upgrade to dcp-cli (aka hca) version 6.4.0 (#91)

HumanCellAtlas · Nov 14, 2019 · 8fb3128 · 8fb3128
1 parent e2ae3ad
commit 8fb3128
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 29 deletions.
diff --git a/setup.py b/setup.py
@@ -10,7 +10,7 @@
     # Not using tests_require because that installs the test requirements into .eggs, not the virtualenv
     extras_require={
         "dss": [
-            'hca == 5.2.0',
+            'hca == 6.4.0',
             'urllib3 >= 1.23',
             'requests >= 2.19.1'
         ],

diff --git a/src/humancellatlas/data/metadata/helpers/dss.py b/src/humancellatlas/data/metadata/helpers/dss.py
@@ -73,19 +73,14 @@ def download_bundle_metadata(client: DSSClient,
                   replica=replica,
                   directurls=directurls,
                   presignedurls=presignedurls)
-    url = None
     manifest = []
-    while True:
-        # We can't use get_file.iterate because it only returns the `bundle.files` part of the response and swallows
-        # the `bundle.version`. See https://github.com/HumanCellAtlas/dcp-cli/issues/331
-        # noinspection PyUnresolvedReferences,PyProtectedMember
-        response = client.get_bundle._request(kwargs, url=url)
-        bundle = response.json()['bundle']
+
+    bundle = None
+    # noinspection PyUnresolvedReferences
+    for page in client.get_bundle.paginate(**kwargs):
+        bundle = page['bundle']
         manifest.extend(bundle['files'])
-        try:
-            url = response.links['next']['url']
-        except KeyError:
-            break
+    assert bundle is not None
 
     metadata_files = {f['name']: f for f in manifest if f['indexed']}
 

diff --git a/test/test.py b/test/test.py
@@ -180,24 +180,23 @@ def _load_bundle(self, uuid, version, replica='aws', deployment='prod'):
         return manifest, metadata_files
 
     def _mock_get_bundle(self, file_uuid, file_version, content_type):
-        response = Mock()
-        response.links = {}
-        response.json.return_value = {
-            'bundle': {
-                'version': '2018-09-20T232924.687620Z',
-                'files': [
-                    {
-                        'name': 'name.json',
-                        'uuid': file_uuid,
-                        'version': file_version,
-                        'indexed': True,
-                        'content-type': content_type
-                    }
-                ]
-            }
-        }
         client = Mock()
-        client.get_bundle._request.return_value = response
+        client.get_bundle.paginate.return_value = [
+            {
+                'bundle': {
+                    'version': '2018-09-20T232924.687620Z',
+                    'files': [
+                        {
+                            'name': 'name.json',
+                            'uuid': file_uuid,
+                            'version': file_version,
+                            'indexed': True,
+                            'content-type': content_type
+                        }
+                    ]
+                }
+            }
+        ]
         return client
 
     def test_bad_content(self):