2024-03-26 nightly release (2b831b3)

.github/workflows/domain_ci.yml

@@ -10,50 +10,6 @@ on:
       - gh/*/*/base
 
 jobs:
-  torchtext:
-    if: ${{ github.repository_owner == 'pytorch' }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - macos-latest
-          - ubuntu-latest
-          - windows-latest
-        python-version:
-          - 3.8
-          - 3.9
-    steps:
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install torch and torchtext from nightlies
-        run: |
-          pip install numpy networkx
-          pip install --pre torch torchtext --index-url https://download.pytorch.org/whl/nightly/cpu
-
-      - name: Check out torchdata repository
-        uses: actions/checkout@v3
-
-      - name: Install torchdata
-        run: |
-          pip install -r requirements.txt
-          python setup.py install
-
-      - name: Install test requirements
-        run: pip install dill expecttest pytest iopath
-
-      - name: Run torchtext example datasets tests
-        if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }}
-        run: pytest --no-header -v test/test_text_examples.py
-      - name: Run torchtext example datasets (including slow tests)
-        if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }}
-        run: pytest --no-header -v test/test_text_examples.py
-        env:
-          PYTORCH_TEST_WITH_SLOW: 1
-
   torchaudio:
     if: ${{ github.repository_owner == 'pytorch' }}
     runs-on: ${{ matrix.os }}

.github/workflows/lint.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip3 install -r requirements.txt
-          pip3 install mypy==0.960 numpy types-requests
+          pip3 install mypy==1.8.0 numpy types-requests
       - name: Build TorchData
         run: |
           pip3 install .

torchdata/datapipes/iter/load/online.py

@@ -36,7 +36,7 @@ def _get_response_from_http(
 ) -> Tuple[str, StreamWrapper]:
     with requests.Session() as session:
         proxies = _get_proxies()
-        r = session.get(url, timeout=timeout, proxies=proxies, stream=True, **query_params)  # type: ignore[attr-defined]
+        r = session.get(url, timeout=timeout, proxies=proxies, stream=True, **query_params)  # type: ignore[arg-type]
     r.raise_for_status()
     return url, StreamWrapper(r.raw)
 
@@ -112,7 +112,7 @@ def _get_response_from_google_drive(
     confirm_token = None
 
     with requests.Session() as session:
-        response = session.get(url, timeout=timeout, stream=True, **query_params)  # type: ignore[attr-defined]
+        response = session.get(url, timeout=timeout, stream=True, **query_params)  # type: ignore[arg-type]
         response.raise_for_status()
 
         for k, v in response.cookies.items():
@@ -129,7 +129,7 @@ def _get_response_from_google_drive(
         if confirm_token:
             url = url + "&confirm=" + confirm_token
 
-        response = session.get(url, timeout=timeout, stream=True, **query_params)  # type: ignore[attr-defined]
+        response = session.get(url, timeout=timeout, stream=True, **query_params)  # type: ignore[arg-type]
         response.raise_for_status()
 
         if "content-disposition" not in response.headers:

torchdata/datapipes/iter/util/cacheholder.py

@@ -202,7 +202,7 @@ def __init__(
         self,
         source_datapipe: IterDataPipe,
         filepath_fn: Optional[Callable] = None,
-        hash_dict: Dict[str, str] = None,
+        hash_dict: Optional[Dict[str, str]] = None,
         hash_type: str = "sha256",
         extra_check_fn: Optional[Callable[[str], bool]] = None,
     ):
@@ -258,8 +258,7 @@ def _cache_check_fn(data, filepath_fn, hash_dict, hash_type, extra_check_fn, cac
         if not cached_file_exists:
             promise_filepath = _promise_filename(filepath, cache_uuid)
             dirname = os.path.dirname(promise_filepath)
-            if not os.path.exists(dirname):
-                os.makedirs(dirname)
+            os.makedirs(dirname, exist_ok=True)
 
             with portalocker.Lock(promise_filepath, "a+", flags=portalocker.LockFlags.EXCLUSIVE) as promise_fh:
                 promise_fh.seek(0)

torchdata/datapipes/iter/util/rows2columnar.py

@@ -5,7 +5,7 @@
 # LICENSE file in the root directory of this source tree.
 
 from collections import defaultdict
-from typing import Dict, Iterator, List, Union
+from typing import Dict, Iterator, List, Optional, Union
 
 from torchdata.datapipes import functional_datapipe
 from torchdata.datapipes.iter import IterDataPipe
@@ -50,7 +50,9 @@ class Rows2ColumnarIterDataPipe(IterDataPipe[Dict]):
     """
     column_names: List[str]
 
-    def __init__(self, source_datapipe: IterDataPipe[List[Union[Dict, List]]], column_names: List[str] = None) -> None:
+    def __init__(
+        self, source_datapipe: IterDataPipe[List[Union[Dict, List]]], column_names: Optional[List[str]] = None
+    ) -> None:
         self.source_datapipe: IterDataPipe[List[Union[Dict, List]]] = source_datapipe
         self.column_names: List[str] = [] if column_names is None else column_names